Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

Flow dictation: keep the last word and the whole sentence

Claude362c7a797d33
diff --git a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
index 171a3401..869124c6 100644
--- a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
+++ b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
@@ -47,6 +47,7 @@
1C2BB61FA83015C42B321E0E /* PillColors.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37564F5F041B38C976DF3DEF /* PillColors.swift */; };
1D23613B270FE8B11C5E27A7 /* IslandDisplayChoice.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A09E73F1E58019EF6EA2A62 /* IslandDisplayChoice.swift */; };
1D4CE0CCD2B5A0FFE50C7F06 /* TerminalTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */; };
+ 5663BA3BFBE656C2F5C00290 /* DictationTranscript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */; };
0AF77F5721986BE11151CE87 /* FlowSettingsBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */; };
614F9783757FE777EF4CABB5 /* FlowPill.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79D06C32348E26EAA1087BFC /* FlowPill.swift */; };
F05CD98E6BF8679F12880345 /* FlowDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17870B121906BE24A935C407 /* FlowDictation.swift */; };
@@ -132,6 +133,7 @@
65CB53E20B2289CA3AC3E8FE /* TurnSnapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A1C4152EBDBD7C24CABFFD7 /* TurnSnapshot.swift */; };
6613EB914852E6EEB1FF37BF /* IntentParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = F39753CFF90B494D449CCD95 /* IntentParser.swift */; };
6726F74966EDE7F82BFE89B2 /* TerminalTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */; };
+ AAC6AB081F959CEE5FE628C8 /* DictationTranscript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */; };
98863553DD449BD791A1E00C /* FlowSettingsBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */; };
15C9E266B443EFE70908510B /* FlowPill.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79D06C32348E26EAA1087BFC /* FlowPill.swift */; };
8BF7CB586067C38AB5C4738C /* FlowDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17870B121906BE24A935C407 /* FlowDictation.swift */; };
@@ -470,6 +472,7 @@
871CA2BADC67D97C97C7009D /* error.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = error.wav; sourceTree = "<group>"; };
8790ACE9F73BD74640A0F20C /* LocalChat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalChat.swift; sourceTree = "<group>"; };
87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalTarget.swift; sourceTree = "<group>"; };
+ 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationTranscript.swift; sourceTree = "<group>"; };
0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowSettingsBox.swift; sourceTree = "<group>"; };
79D06C32348E26EAA1087BFC /* FlowPill.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowPill.swift; sourceTree = "<group>"; };
17870B121906BE24A935C407 /* FlowDictation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowDictation.swift; sourceTree = "<group>"; };
@@ -630,6 +633,7 @@
914A792788D403F836BEB570 /* SpotifyViews.swift */,
2EF78999D68D56ADE4CEFBE2 /* StripePoller.swift */,
87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */,
+ 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */,
0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */,
79D06C32348E26EAA1087BFC /* FlowPill.swift */,
17870B121906BE24A935C407 /* FlowDictation.swift */,
@@ -1118,6 +1122,7 @@
1558BB00C530B721172F6E65 /* StripePoller.swift in Sources */,
9EA178EA395B116336C1FA10 /* TerminalTarget+AppKit.swift in Sources */,
1D4CE0CCD2B5A0FFE50C7F06 /* TerminalTarget.swift in Sources */,
+ 5663BA3BFBE656C2F5C00290 /* DictationTranscript.swift in Sources */,
0AF77F5721986BE11151CE87 /* FlowSettingsBox.swift in Sources */,
614F9783757FE777EF4CABB5 /* FlowPill.swift in Sources */,
F05CD98E6BF8679F12880345 /* FlowDictation.swift in Sources */,
@@ -1324,6 +1329,7 @@
E4B9986674F4D425826F15A8 /* StripePoller.swift in Sources */,
11F14DB97C93CF84DF9D5C43 /* TerminalTarget+AppKit.swift in Sources */,
6726F74966EDE7F82BFE89B2 /* TerminalTarget.swift in Sources */,
+ AAC6AB081F959CEE5FE628C8 /* DictationTranscript.swift in Sources */,
98863553DD449BD791A1E00C /* FlowSettingsBox.swift in Sources */,
15C9E266B443EFE70908510B /* FlowPill.swift in Sources */,
8BF7CB586067C38AB5C4738C /* FlowDictation.swift in Sources */,
diff --git a/NotchBuddy/Sources/App/DictationTranscript.swift b/NotchBuddy/Sources/App/DictationTranscript.swift
new file mode 100644
index 00000000..cc85469d
--- /dev/null
+++ b/NotchBuddy/Sources/App/DictationTranscript.swift
@@ -0,0 +1,84 @@
+import Foundation
+
+/// The words of one dictation, kept whole when Apple's recognizer loses some of them
+/// (tests/FlowKeyLogicTests.swift).
+///
+/// On a long dictation the recognizer can:
+/// - start over after a pause: the next result holds only the new words;
+/// - drop a stretch it had already given, in the middle of a later result.
+/// Taken as is, either loses what was said. So each result is merged with the words seen before:
+/// a result that looks like a fresh start closes the text so far, and a word heard earlier that no
+/// newer word covers in time is kept where it was.
+struct DictationTranscript: Equatable {
+ /// One word (or a few) with its place in the audio, in seconds.
+ struct Piece: Equatable, Sendable {
+ var start: Double
+ var end: Double
+ var text: String
+ }
+
+ /// Text closed by a fresh start of the recognizer.
+ private(set) var committed = ""
+ /// The words of the stretch in progress.
+ private(set) var pieces: [Piece] = []
+ /// Their text: the recognizer's own (formatted) string, unless words had to be put back.
+ private(set) var current = ""
+
+ var text: String { Self.join(committed, current) }
+
+ /// A newer word closer than this to an older one, in time, replaces it.
+ static let tolerance = 0.25
+
+ /// - Parameters:
+ /// - formatted: the result's text as the recognizer wrote it.
+ /// - pieces: its words with their times (all zero when the recognizer gives no times).
+ mutating func absorb(_ formatted: String, pieces new: [Piece]) {
+ if Self.isRestart(previous: current, new: formatted) {
+ committed = Self.join(committed, current)
+ pieces = new
+ current = formatted
+ return
+ }
+ let timed = !new.isEmpty && new.contains { $0.end > $0.start }
+ guard timed, let last = new.last else {
+ pieces = new
+ current = formatted
+ return
+ }
+ // Earlier words the new result left out, that no new word covers: put them back.
+ let kept = pieces.filter { old in
+ old.end <= last.start + Self.tolerance
+ && !new.contains { Self.close(old, $0) }
+ }
+ if kept.isEmpty {
+ pieces = new
+ current = formatted
+ } else {
+ pieces = (kept + new).sorted { $0.start < $1.start }
+ current = pieces.map(\.text).joined(separator: " ")
+ }
+ }
+
+ /// Plain text only (no times): a fresh start closes the text so far, anything else replaces it.
+ mutating func absorb(_ formatted: String) { absorb(formatted, pieces: []) }
+
+ /// A result is a fresh start when the text so far had a few words, the new one has less than half
+ /// as many, and it does not start with the same word. A correction of the last words, or a
+ /// recognizer that keeps everything, never looks like that.
+ static func isRestart(previous: String, new: String) -> Bool {
+ let p = previous.split(whereSeparator: \.isWhitespace)
+ let n = new.split(whereSeparator: \.isWhitespace)
+ guard p.count >= 3, !n.isEmpty, n.count * 2 <= p.count else { return false }
+ return p[0].lowercased() != n[0].lowercased()
+ }
+
+ private static func close(_ a: Piece, _ b: Piece) -> Bool {
+ a.start < b.end + tolerance && b.start < a.end + tolerance
+ }
+
+ private static func join(_ a: String, _ b: String) -> String {
+ if a.isEmpty { return b }
+ if b.isEmpty { return a }
+ return a + " " + b
+ }
+}
diff --git a/NotchBuddy/Sources/App/FlowDictation.swift b/NotchBuddy/Sources/App/FlowDictation.swift
index 7c4c1529..a78cea40 100644
--- a/NotchBuddy/Sources/App/FlowDictation.swift
+++ b/NotchBuddy/Sources/App/FlowDictation.swift
@@ -59,6 +59,8 @@ final class FlowDictation: ObservableObject {
static let shared = FlowDictation()
static let enabledKey = "flowDictationEnabled"
+ /// How long the microphone stays open after the key comes up.
+ static let tailMilliseconds = 600
static let keyKey = "flowDictationKey"
@Published var isEnabled: Bool = UserDefaults.standard.bool(forKey: FlowDictation.enabledKey) {
@@ -237,6 +239,10 @@ final class FlowDictation: ObservableObject {
let opening = startTask
Task { @MainActor [weak self] in
await opening?.value
+ // The key often comes up while the last word is still being said: keep listening a
+ // moment so it makes it in.
+ try? await Task.sleep(for: .milliseconds(Self.tailMilliseconds))
+ guard !listen.cancelled else { return }
let text = await listen.dictation.finish().trimmingCharacters(in: .whitespacesAndNewlines)
guard let self, !listen.cancelled else { return }
// A new listen may have begun meanwhile: its pill is left alone, but these words still go in.
diff --git a/NotchBuddy/Sources/App/MacDictation.swift b/NotchBuddy/Sources/App/MacDictation.swift
index f9bc29b8..07838bc7 100644
--- a/NotchBuddy/Sources/App/MacDictation.swift
+++ b/NotchBuddy/Sources/App/MacDictation.swift
@@ -49,7 +49,9 @@ final class MacDictation: @unchecked Sendable {
let recognizer: SFSpeechRecognizer
let request = SFSpeechAudioBufferRecognitionRequest()
var task: SFSpeechRecognitionTask?
- var words = ""
+ /// What was heard, kept whole when the recognizer starts over after a pause.
+ var heard = DictationTranscript()
+ var words: String { heard.text }
/// Mean confidence of the words (Apple fills it in on the final result).
var confidence: Float = 0
var finished = false
@@ -76,9 +78,9 @@ final class MacDictation: @unchecked Sendable {
isRecording = true
NotificationCenter.default.post(name: .dictationDidStart, object: nil)
for (index, candidate) in built.enumerated() {
- candidate.task = Self.recognize(candidate) { [weak self] words, confidence, final, failed in
+ candidate.task = Self.recognize(candidate) { [weak self] words, pieces, confidence, final, failed in
Task { @MainActor in
- self?.update(index, words: words, confidence: confidence, final: final, failed: failed)
+ self?.update(index, words: words, pieces: pieces, confidence: confidence, final: final, failed: failed)
}
}
}
@@ -90,12 +92,13 @@ final class MacDictation: @unchecked Sendable {
isFinishing = true
stopEngine()
candidates.forEach { $0.request.endAudio() }
- // With several languages, wait a moment for the final results: they carry the confidence.
- if candidates.count > 1, !candidates.allSatisfy(\.finished) {
+ // Wait a moment for the final results: they carry the last words (and, with several
+ // languages, the confidence that picks one).
+ if !candidates.allSatisfy(\.finished) {
await withCheckedContinuation { continuation in
finishing = continuation
Task { @MainActor [weak self] in
- try? await Task.sleep(for: .milliseconds(1200))
+ try? await Task.sleep(for: .milliseconds(1500))
self?.resumeFinishing()
}
}
@@ -140,10 +143,10 @@ final class MacDictation: @unchecked Sendable {
// MARK: Results
- @MainActor private func update(_ index: Int, words: String?, confidence: Float, final: Bool, failed: Bool) {
+ @MainActor private func update(_ index: Int, words: String?, pieces: [DictationTranscript.Piece], confidence: Float, final: Bool, failed: Bool) {
guard index < candidates.count else { return }
let candidate = candidates[index]
- if let words { candidate.words = words }
+ if let words { candidate.heard.absorb(words, pieces: pieces) }
if confidence > 0 { candidate.confidence = confidence }
if final || failed { candidate.finished = true }
@@ -197,13 +200,16 @@ final class MacDictation: @unchecked Sendable {
nonisolated private static func recognize(
_ candidate: Candidate,
- report: @escaping @Sendable (String?, Float, Bool, Bool) -> Void
+ report: @escaping @Sendable (String?, [DictationTranscript.Piece], Float, Bool, Bool) -> Void
) -> SFSpeechRecognitionTask {
candidate.recognizer.recognitionTask(with: candidate.request) { result, error in
let words = result?.bestTranscription.formattedString
let segments = result?.bestTranscription.segments ?? []
let confidence = segments.isEmpty ? 0 : segments.map(\.confidence).reduce(0, +) / Float(segments.count)
- report(words, confidence, result?.isFinal ?? false, error != nil)
+ let pieces = segments.map {
+ DictationTranscript.Piece(start: $0.timestamp, end: $0.timestamp + $0.duration, text: $0.substring)
+ }
+ report(words, pieces, confidence, result?.isFinal ?? false, error != nil)
}
}
diff --git a/scripts/test-flow-keys.sh b/scripts/test-flow-keys.sh
index 91b09ca5..765f8886 100755
--- a/scripts/test-flow-keys.sh
+++ b/scripts/test-flow-keys.sh
@@ -3,6 +3,6 @@ set -euo pipefail
cd "$(dirname "$0")/.."
TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/coucou-flow-keys.XXXXXX")"
trap 'rm -rf "$TEST_DIR"' EXIT
-swiftc NotchBuddy/Sources/App/FlowKeyLogic.swift \
+swiftc NotchBuddy/Sources/App/FlowKeyLogic.swift NotchBuddy/Sources/App/DictationTranscript.swift \
tests/FlowKeyLogicTests.swift -o "$TEST_DIR/flow-keys-tests"
"$TEST_DIR/flow-keys-tests"
diff --git a/tests/FlowKeyLogicTests.swift b/tests/FlowKeyLogicTests.swift
index 6328bc94..746af5e9 100644
--- a/tests/FlowKeyLogicTests.swift
+++ b/tests/FlowKeyLogicTests.swift
@@ -89,6 +89,54 @@ enum FlowKeyLogicTests {
check("in between", FlowBands.level(db: -54) > 0.3 && FlowBands.level(db: -54) < 0.7)
}
+ print("\nTranscript kept whole")
+ do {
+ typealias P = DictationTranscript.Piece
+ func words(_ list: [(Double, String)]) -> [P] { list.map { P(start: $0.0, end: $0.0 + 0.3, text: $0.1) } }
+
+ var t = DictationTranscript()
+ t.absorb("Ok mais", pieces: words([(0, "Ok"), (0.4, "mais")]))
+ t.absorb("Ok mais c'est au top", pieces: words([(0, "Ok"), (0.4, "mais"), (0.8, "c'est"), (1.2, "au"), (1.6, "top")]))
+ check("a growing result is taken as is", t.text == "Ok mais c'est au top")
+ t.absorb("Ok mais c'est au top.", pieces: words([(0, "Ok"), (0.4, "mais"), (0.8, "c'est"), (1.2, "au"), (1.6, "top.")]))
+ check("the recognizer's own formatting wins", t.text == "Ok mais c'est au top.")
+
+ // The recognizer drops the middle stretch from a later result.
+ var m = DictationTranscript()
+ m.absorb("un deux trois quatre cinq", pieces: words([(0, "un"), (1, "deux"), (2, "trois"), (3, "quatre"), (4, "cinq")]))
+ m.absorb("un deux cinq six", pieces: words([(0, "un"), (1, "deux"), (4, "cinq"), (5, "six")]))
+ check("a dropped middle stretch is put back", m.text == "un deux trois quatre cinq six")
+
+ // A correction of the last words replaces them, nothing is doubled.
+ var c = DictationTranscript()
+ c.absorb("je vais à la plage", pieces: words([(0, "je"), (0.5, "vais"), (1, "à"), (1.3, "la"), (1.6, "plage")]))
+ c.absorb("je vais à Laplace", pieces: words([(0, "je"), (0.5, "vais"), (1, "à"), (1.35, "Laplace")]))
+ check("a correction replaces the old words", c.text == "je vais à Laplace")
+
+ // Times that shift a little between results do not double a word.
+ var s = DictationTranscript()
+ s.absorb("bonjour à tous", pieces: words([(1.0, "bonjour"), (1.4, "à"), (1.7, "tous")]))
+ s.absorb("Bonjour à tous", pieces: words([(1.15, "Bonjour"), (1.5, "à"), (1.8, "tous")]))
+ check("shifted times replace, not double", s.text == "Bonjour à tous")
+
+ // The recognizer starts over after a pause, times from zero again.
+ var r = DictationTranscript()
+ r.absorb("la première phrase est assez longue", pieces: words([(0, "la"), (0.4, "première"), (0.9, "phrase"), (1.3, "est"), (1.6, "assez"), (2.0, "longue")]))
+ r.absorb("ensuite", pieces: words([(0, "ensuite")]))
+ r.absorb("ensuite la suite", pieces: words([(0, "ensuite"), (0.5, "la"), (0.8, "suite")]))
+ check("a fresh start keeps what came before", r.text == "la première phrase est assez longue ensuite la suite")
+
+ // No times at all: plain replace, fresh starts still kept.
+ var u = DictationTranscript()
+ u.absorb("hello there my friend")
+ u.absorb("hello there my friend how are you")
+ check("untimed growth", u.text == "hello there my friend how are you")
+ u.absorb("great")
+ check("untimed fresh start", u.text == "hello there my friend how are you great")
+ check("a short correction is not a fresh start", !DictationTranscript.isRestart(previous: "one two three four", new: "one two"))
+ check("too little text to call it a fresh start", !DictationTranscript.isRestart(previous: "one two", new: "three"))
+ }
+
if failures == 0 {
print("\nAll tests passed.")
exit(0)