Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

Notetaker: remove the echo by the sound, merge a voice cut in two, correct misheard words

Claudec6276017a69a
diff --git a/NotchBuddy/Resources/Localizable.xcstrings b/NotchBuddy/Resources/Localizable.xcstrings
index 1170e931..cf5d3197 100644
--- a/NotchBuddy/Resources/Localizable.xcstrings
+++ b/NotchBuddy/Resources/Localizable.xcstrings
@@ -1,6 +1,396 @@
{
"sourceLanguage" : "en",
"strings" : {
+ "notes.correct-again" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "تصحيح النص المكتوب من جديد"
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "ট্রান্সক্রিপ্ট আবার ঠিক করুন"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Correct the transcript again"
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corregir de nuevo la transcripción"
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corriger à nouveau la transcription"
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "ट्रांसक्रिप्ट फिर से सुधारें"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Koreksi transkrip lagi"
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corrigir a transcrição de novo"
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Исправить расшифровку заново"
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "重新校正转录"
+ }
+ }
+ }
+ },
+ "notes.delete-line" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "حذف السطر"
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "লাইনটি মুছুন"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Delete the line"
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Eliminar la línea"
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Supprimer la ligne"
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "पंक्ति हटाएँ"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Hapus baris"
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Apagar a linha"
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Удалить строку"
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "删除这一行"
+ }
+ }
+ }
+ },
+ "notes.edit-line" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "تصحيح السطر"
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "লাইনটি ঠিক করুন"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Fix the line"
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corregir la línea"
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corriger la ligne"
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "पंक्ति ठीक करें"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Perbaiki baris"
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Corrigir a linha"
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Исправить строку"
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "修改这一行"
+ }
+ }
+ }
+ },
+ "notes.edit-line-detail" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "تُحفظ الكلمات التي تصححها لمكالماتك القادمة."
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "আপনি যে শব্দ ঠিক করেন, তা পরের কলগুলোর জন্য মনে রাখা হয়।"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "The words you bring in are remembered for your next calls."
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Las palabras que corriges se recuerdan para tus próximas llamadas."
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Les mots que tu corriges sont retenus pour tes prochains appels."
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "आप जो शब्द सुधारते हैं, वे अगली कॉल के लिए याद रखे जाते हैं।"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Kata yang kamu perbaiki diingat untuk panggilan berikutnya."
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "As palavras que você corrige ficam guardadas para as próximas chamadas."
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Исправленные слова запоминаются для следующих звонков."
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "你改正的词会被记住,用于之后的通话。"
+ }
+ }
+ }
+ },
+ "notes.said-by" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "قالها"
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "কে বলেছেন"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Said by"
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Dicho por"
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Dit par"
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "किसने कहा"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Diucapkan oleh"
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Dito por"
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Сказал(а)"
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "说话人"
+ }
+ }
+ }
+ },
+ "notes.transcript-hint" : {
+ "extractionState" : "manual",
+ "localizations" : {
+ "ar" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "كلمة خاطئة أو شخص خاطئ؟ صحّحها من قائمة ⋯ في السطر. تُحفظ الكلمات التي تصححها لمكالماتك القادمة."
+ }
+ },
+ "bn" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "ভুল শব্দ, নাকি ভুল ব্যক্তি? লাইনের ⋯ মেনু থেকে ঠিক করুন। আপনি যে শব্দ ঠিক করেন, তা পরের কলগুলোর জন্য মনে রাখা হয়।"
+ }
+ },
+ "en" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "A wrong word, or the wrong person? Fix it from the line's ⋯ menu. Words you correct are remembered for your next calls."
+ }
+ },
+ "es" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "¿Una palabra mal o la persona equivocada? Corrígelo desde el menú ⋯ de la línea. Las palabras que corriges se recuerdan para tus próximas llamadas."
+ }
+ },
+ "fr" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Un mot faux, ou la mauvaise personne ? Corrige-le depuis le menu ⋯ de la ligne. Les mots que tu corriges sont retenus pour tes prochains appels."
+ }
+ },
+ "hi" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "कोई शब्द गलत है, या व्यक्ति गलत? पंक्ति के ⋯ मेनू से ठीक करें। आप जो शब्द सुधारते हैं, वे अगली कॉल के लिए याद रखे जाते हैं।"
+ }
+ },
+ "id" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Kata yang salah, atau orang yang salah? Perbaiki dari menu ⋯ di baris itu. Kata yang kamu perbaiki diingat untuk panggilan berikutnya."
+ }
+ },
+ "pt-BR" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Uma palavra errada ou a pessoa errada? Corrija pelo menu ⋯ da linha. As palavras que você corrige ficam guardadas para as próximas chamadas."
+ }
+ },
+ "ru" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "Неверное слово или не тот человек? Исправьте через меню ⋯ строки. Исправленные слова запоминаются для следующих звонков."
+ }
+ },
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "词听错了,或说话人不对?在该行的 ⋯ 菜单里修改。你改正的词会被记住,用于之后的通话。"
+ }
+ }
+ }
+ },
"vitals.title" : {
"extractionState" : "manual",
"localizations" : {
diff --git a/NotchBuddy/Sources/App/FlowHubNotes.swift b/NotchBuddy/Sources/App/FlowHubNotes.swift
index 7bd71a57..584bec4a 100644
--- a/NotchBuddy/Sources/App/FlowHubNotes.swift
+++ b/NotchBuddy/Sources/App/FlowHubNotes.swift
@@ -357,6 +357,8 @@ private struct HubNoteDetail: View {
@State private var renaming: NoteSpeaker?
@State private var newName = ""
@State private var confirmDelete = false
+ @State private var editing: NoteLine?
+ @State private var editText = ""
var body: some View {
ScrollView {
@@ -399,6 +401,16 @@ private struct HubNoteDetail: View {
}
Button(String(localized: "hub.cancel"), role: .cancel) { renaming = nil }
}
+ .alert(String(localized: "notes.edit-line"), isPresented: Binding(get: { editing != nil }, set: { if !$0 { editing = nil } })) {
+ TextField("", text: $editText)
+ Button(String(localized: "hub.save")) {
+ if let line = editing { taker.edit(note.id, line: line.id, text: editText) }
+ editing = nil
+ }
+ Button(String(localized: "hub.cancel"), role: .cancel) { editing = nil }
+ } message: {
+ Text("notes.edit-line-detail")
+ }
.alert(String(localized: "notes.delete-confirm"), isPresented: $confirmDelete) {
Button(String(localized: "hub.delete"), role: .destructive) { taker.delete(note.id) }
Button(String(localized: "hub.cancel"), role: .cancel) {}
@@ -436,6 +448,8 @@ private struct HubNoteDetail: View {
Button(String(localized: "notes.export")) { taker.export(note) }
Button(String(localized: "notes.summarize-again")) { Task { await taker.summarize(note.id) } }
.disabled(taker.summaryEngine == .off || note.lines.isEmpty)
+ Button(String(localized: "notes.correct-again")) { Task { await taker.correctAgain(note.id) } }
+ .disabled(!taker.canCorrect || note.lines.isEmpty || taker.working.contains(note.id))
Divider()
}
Button(String(localized: "hub.delete"), role: .destructive) { confirmDelete = true }
@@ -570,11 +584,64 @@ private struct HubNoteDetail: View {
.font(.system(size: 13))
.foregroundColor(HubTheme.secondary)
}
+ if !note.lines.isEmpty {
+ Text("notes.transcript-hint")
+ .font(.system(size: 12))
+ .foregroundColor(HubTheme.muted)
+ }
ForEach(note.lines) { line in
let speaker = note.speaker(line.speaker)
- HubLineView(name: speaker?.name ?? "?", color: HubTheme.speaker(speaker?.color ?? 1), line: line)
+ HubEditableLine(note: note, line: line, name: speaker?.name ?? "?",
+ color: HubTheme.speaker(speaker?.color ?? 1)) {
+ editText = line.text
+ editing = line
+ }
+ }
+ }
+ }
+}
+
+/// A line of a noted call, with what can be fixed on it: its words, who said it, or the line itself
+/// (an echo left over). From the line's menu (shown on hover) or a right-click.
+private struct HubEditableLine: View {
+ let note: Note
+ let line: NoteLine
+ let name: String
+ let color: Color
+ let onEdit: () -> Void
+ @ObservedObject private var taker = NoteTaker.shared
+ @State private var hovering = false
+
+ var body: some View {
+ HubLineView(name: name, color: color, line: line)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .contentShape(Rectangle())
+ .overlay(alignment: .topTrailing) {
+ Menu { actions } label: {
+ Image(systemName: "ellipsis")
+ .font(.system(size: 12))
+ .foregroundColor(HubTheme.secondary)
+ }
+ .menuStyle(.borderlessButton)
+ .menuIndicator(.hidden)
+ .fixedSize()
+ .opacity(hovering ? 1 : 0)
+ .help(String(localized: "notes.edit-line"))
+ }
+ .onHover { hovering = $0 }
+ .contextMenu { actions }
+ }
+
+ @ViewBuilder private var actions: some View {
+ Button(String(localized: "notes.edit-line"), action: onEdit)
+ Menu(String(localized: "notes.said-by")) {
+ ForEach(note.speakers) { other in
+ Button(other.name) { taker.assign(note.id, line: line.id, to: other.id) }
+ .disabled(other.id == line.speaker)
}
}
+ Divider()
+ Button(String(localized: "notes.delete-line"), role: .destructive) { taker.deleteLine(note.id, line: line.id) }
}
}
diff --git a/NotchBuddy/Sources/App/NoteAudio.swift b/NotchBuddy/Sources/App/NoteAudio.swift
index 0565ab71..89819bfb 100644
--- a/NotchBuddy/Sources/App/NoteAudio.swift
+++ b/NotchBuddy/Sources/App/NoteAudio.swift
@@ -22,11 +22,16 @@ final class ChannelRecorder: @unchecked Sendable {
var level: Float = 0
/// Seconds with sound in them, to tell an empty side (no call, no headphones) from a real one.
var voiced = 0.0
+ var started = false
}
private let live = OSAllocatedUnfairLock(initialState: Live())
- init(url: URL, label: String) throws {
+ /// When the recording started (`DispatchTime` uptime): both sides count their time from it.
+ private let origin: UInt64
+
+ init(url: URL, label: String, origin: UInt64 = DispatchTime.now().uptimeNanoseconds) throws {
self.url = url
+ self.origin = origin
queue = DispatchQueue(label: label, qos: .userInitiated)
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatLinearPCM,
@@ -42,6 +47,19 @@ final class ChannelRecorder: @unchecked Sendable {
/// From the audio thread.
func append(_ samples: [Float]) {
guard !samples.isEmpty else { return }
+ let incoming = Double(samples.count) / Self.sampleRate
+ let origin = self.origin
+ // The Mac's sound starts a moment after the microphone (its device takes longer to open):
+ // the silence before its first sound is written, so both recordings share one clock and
+ // the echo and the order of the lines line up.
+ let lead = live.withLock { state -> Int in
+ guard !state.started else { return 0 }
+ state.started = true
+ let elapsed = Double(DispatchTime.now().uptimeNanoseconds &- origin) / 1_000_000_000
+ let seconds = min(10, elapsed - incoming)
+ return seconds > 0.01 ? Int(seconds * Self.sampleRate) : 0
+ }
+ let samples = lead > 0 ? [Float](repeating: 0, count: lead) + samples : samples
let rms = Self.rms(samples[...])
live.withLock { state in
state.pending.append(contentsOf: samples)
diff --git a/NotchBuddy/Sources/App/NoteEcho.swift b/NotchBuddy/Sources/App/NoteEcho.swift
new file mode 100644
index 00000000..41823d87
--- /dev/null
+++ b/NotchBuddy/Sources/App/NoteEcho.swift
@@ -0,0 +1,145 @@
+import Foundation
+
+/// Without headphones, the others' voices come out of the speakers and back into the microphone,
+/// and Parakeet writes them as yours (often heard a little differently, so comparing the words is
+/// not enough). The Mac's own sound is the reference: where the microphone holds no more energy
+/// than the echo of that sound explains, what it heard is the others, not you.
+/// Pure functions on 16 kHz samples (tests/NoteModelsTests.swift).
+enum NoteEcho {
+ /// Levels are measured on 20 ms frames.
+ static let frameSeconds = 0.02
+ /// How much louder than the echo the microphone must be for a word to be yours (≈ 10 dB).
+ static let margin: Float = 3
+ /// The microphone lags behind what the Mac plays: searched from -100 ms to +500 ms.
+ static let lags = -5...25
+
+ struct Analysis: Sendable {
+ /// Frames the microphone lags behind the Mac's sound.
+ var lag: Int
+ /// The microphone's level for one unit of the Mac's sound: the echo's strength (≈ 0 with headphones).
+ var gain: Float
+ /// Below this, the Mac plays nothing.
+ var sysActive: Float
+ /// The microphone's level in silence.
+ var micFloor: Float
+ var mic: [Float]
+ var sys: [Float]
+ }
+
+ /// RMS level of each 20 ms frame.
+ static func levels(_ samples: [Float], sampleRate: Double = 16_000) -> [Float] {
+ let size = Int(sampleRate * frameSeconds)
+ guard size > 0, !samples.isEmpty else { return [] }
+ var out: [Float] = []
+ out.reserveCapacity(samples.count / size + 1)
+ var index = 0
+ while index < samples.count {
+ let end = min(samples.count, index + size)
+ var sum: Float = 0
+ for k in index..<end { sum += samples[k] * samples[k] }
+ out.append((sum / Float(end - index)).squareRoot())
+ index = end
+ }
+ return out
+ }
+
+ static func percentile(_ values: [Float], _ p: Double) -> Float {
+ guard !values.isEmpty else { return 0 }
+ let sorted = values.sorted()
+ let index = min(sorted.count - 1, max(0, Int(Double(sorted.count - 1) * p)))
+ return sorted[index]
+ }
+
+ /// How the Mac's sound reaches the microphone: the delay and the strength of the echo.
+ /// nil when the Mac played too little sound to tell.
+ static func analyze(mic: [Float], sys: [Float]) -> Analysis? {
+ guard !mic.isEmpty, !sys.isEmpty else { return nil }
+ let sysActive = max(0.004, percentile(sys, 0.1) * 4)
+ let active = sys.indices.filter { sys[$0] > sysActive }
+ guard active.count >= 50 else { return nil }
+ let micFloor = percentile(mic, 0.1)
+
+ // The delay: where the microphone's levels follow the Mac's best.
+ var best = (lag: 0, corr: -Float.infinity)
+ for lag in lags {
+ var xs: [Float] = [], ys: [Float] = []
+ for t in active where t + lag >= 0 && t + lag < mic.count {
+ xs.append(sys[t])
+ ys.append(mic[t + lag])
+ }
+ guard xs.count >= 50 else { continue }
+ let corr = pearson(xs, ys)
+ if corr > best.corr { best = (lag, corr) }
+ }
+
+ // The echo's strength: while only the others speak, the microphone is the Mac's sound times
+ // this. When you speak over them the ratio is far higher, so a low percentile skips those moments.
+ var ratios: [Float] = []
+ for t in active where t + best.lag >= 0 && t + best.lag < mic.count {
+ ratios.append(mic[t + best.lag] / sys[t])
+ }
+ guard ratios.count >= 50 else { return nil }
+ return Analysis(lag: best.lag, gain: percentile(ratios, 0.35), sysActive: sysActive,
+ micFloor: micFloor, mic: mic, sys: sys)
+ }
+
+ /// For one frame of the microphone: nil when the Mac plays nothing then, true when the
+ /// microphone holds nothing more than the echo.
+ static func isEcho(frame f: Int, in a: Analysis) -> Bool? {
+ let t = f - a.lag
+ guard f >= 0, f < a.mic.count, t >= 0, t < a.sys.count, a.sys[t] > a.sysActive else { return nil }
+ return a.mic[f] <= margin * a.gain * a.sys[t] + 2 * a.micFloor
+ }
+
+ /// Which of the microphone's words are the echo of the others.
+ static func echoWords(_ words: [TimedWord], analysis a: Analysis) -> [Bool] {
+ var flags = words.map { word -> Bool in
+ let first = max(0, Int(((word.start - 0.04) / frameSeconds).rounded(.down)))
+ let last = max(first, Int(((word.end + 0.04) / frameSeconds).rounded(.up)))
+ var active = 0, echo = 0
+ for f in first...last {
+ guard let isEcho = isEcho(frame: f, in: a) else { continue }
+ active += 1
+ if isEcho { echo += 1 }
+ }
+ let frames = last - first + 1
+ return active * 2 >= frames && Double(echo) >= Double(active) * 0.6
+ }
+ // A lone word in the middle of the other kind, close to both neighbours, goes with them.
+ let original = flags
+ if words.count >= 3 {
+ for i in 1..<(words.count - 1) where original[i - 1] == original[i + 1] && original[i] != original[i - 1] {
+ if words[i].start - words[i - 1].end < 0.6, words[i + 1].start - words[i].end < 0.6 {
+ flags[i] = original[i - 1]
+ }
+ }
+ }
+ return flags
+ }
+
+ /// The microphone's words without the echo, from both recordings' levels.
+ static func removing(_ words: [TimedWord], micLevels: [Float], sysLevels: [Float]) -> (words: [TimedWord], removed: Int) {
+ guard let analysis = analyze(mic: micLevels, sys: sysLevels) else { return (words, 0) }
+ let flags = echoWords(words, analysis: analysis)
+ let kept = zip(words, flags).filter { !$0.1 }.map(\.0)
+ return (kept, words.count - kept.count)
+ }
+
+ static func removing(_ words: [TimedWord], micSamples: [Float], sysSamples: [Float]) -> (words: [TimedWord], removed: Int) {
+ removing(words, micLevels: levels(micSamples), sysLevels: levels(sysSamples))
+ }
+
+ private static func pearson(_ xs: [Float], _ ys: [Float]) -> Float {
+ let n = Float(xs.count)
+ let mx = xs.reduce(0, +) / n, my = ys.reduce(0, +) / n
+ var sxy: Float = 0, sxx: Float = 0, syy: Float = 0
+ for i in xs.indices {
+ let dx = xs[i] - mx, dy = ys[i] - my
+ sxy += dx * dy
+ sxx += dx * dx
+ syy += dy * dy
+ }
+ guard sxx > 0, syy > 0 else { return 0 }
+ return sxy / (sxx * syy).squareRoot()
+ }
+}
diff --git a/NotchBuddy/Sources/App/NoteModels.swift b/NotchBuddy/Sources/App/NoteModels.swift
index 02ff20d2..9e5d5f7a 100644
--- a/NotchBuddy/Sources/App/NoteModels.swift
+++ b/NotchBuddy/Sources/App/NoteModels.swift
@@ -17,6 +17,8 @@ struct NoteLine: Codable, Hashable, Identifiable, Sendable {
var start: Double
var end: Double
var text: String
+ /// The words as Parakeet heard them, kept when the line was corrected (by Claude or by you).
+ var raw: String? = nil
}
struct NoteTodo: Codable, Hashable, Identifiable, Sendable {
@@ -75,6 +77,8 @@ struct SpeakerTurn: Hashable, Sendable {
var speaker: String
var start: Double
var end: Double
+ /// The voice's fingerprint on this stretch (empty when unknown).
+ var embedding: [Float] = []
}
/// Builds the transcript of a call from the two recordings (tests/NoteModelsTests.swift):
@@ -145,6 +149,115 @@ enum NoteBuilder {
s.lowercased().split { !($0.isLetter || $0.isNumber) }.map(String.init)
}
+ /// The echo found word by word: three or more of your words in a row that the others said in
+ /// the same order at the same moment (one word heard differently, or missed, does not break it).
+ /// Catches what the audio check lets through, when the speakers are very loud.
+ static func removingEchoWords(_ mine: [TimedWord], others: [TimedWord],
+ window: Double = 1.2, minRun: Int = 3) -> [TimedWord] {
+ let a = mine.map { tokens($0.text).joined() }
+ let b = others.map { tokens($0.text).joined() }
+ var echo = [Bool](repeating: false, count: mine.count)
+ var i = 0
+ while i < mine.count {
+ var span = 0
+ if !a[i].isEmpty {
+ for j in others.indices where b[j] == a[i] && abs(others[j].start - mine[i].start) <= window {
+ var x = i + 1, y = j + 1, run = 1
+ while x < mine.count && y < others.count {
+ if a[x] == b[y] {
+ run += 1; x += 1; y += 1
+ } else if y + 1 < others.count && a[x] == b[y + 1] {
+ y += 1 // a word the microphone missed
+ } else if x + 1 < mine.count && a[x + 1] == b[y] {
+ x += 1 // a word the microphone heard in more
+ } else if x + 1 < mine.count && y + 1 < others.count && a[x + 1] == b[y + 1] {
+ x += 1; y += 1 // a word heard differently
+ } else {
+ break
+ }
+ }
+ guard run >= minRun else { continue }
+ // The rest of their sentence, heard too differently to match, is the echo as well:
+ // your words that follow without a pause while they are still speaking.
+ var k = max(j, y - 1)
+ while k + 1 < others.count && others[k + 1].start - others[k].end < 0.6 { k += 1 }
+ let until = others[min(k, others.count - 1)].end + 0.3
+ while x < mine.count && mine[x].start <= until && mine[x].start - mine[x - 1].end < 0.6 { x += 1 }
+ span = max(span, x - i)
+ }
+ }
+ if span > 0 {
+ for k in i..<(i + span) { echo[k] = true }
+ i += span
+ } else {
+ i += 1
+ }
+ }
+ return zip(mine, echo).filter { !$0.1 }.map(\.0)
+ }
+
+ /// A voice heard only in short bits ("Yeah.", "Sure.", the end of a sentence) is almost always
+ /// someone already there: such bits are too short for the diarizer to recognise the voice, and
+ /// it makes a new person of them. Each bit goes to the closest real voice (by fingerprint,
+ /// else the nearest in time). A voice is real once it speaks 3 s in a row, or long enough overall.
+ static let fragmentLongest = 3.0
+
+ static func mergingFragments(_ turns: [SpeakerTurn]) -> [SpeakerTurn] {
+ var total: [String: Double] = [:], longest: [String: Double] = [:]
+ for turn in turns {
+ let length = turn.end - turn.start
+ total[turn.speaker, default: 0] += length
+ longest[turn.speaker] = max(longest[turn.speaker] ?? 0, length)
+ }
+ guard total.count > 1 else { return turns }
+ let all = total.values.reduce(0, +)
+ let solid = Set(total.keys.filter { (longest[$0] ?? 0) >= fragmentLongest || (total[$0] ?? 0) >= max(20, all * 0.1) })
+ guard !solid.isEmpty, solid.count < total.count else { return turns }
+
+ // Each real voice's fingerprint: the mean of its turns', longer turns counting more.
+ var centroids: [String: [Float]] = [:]
+ for speaker in solid {
+ var sum: [Float] = []
+ for turn in turns where turn.speaker == speaker && !turn.embedding.isEmpty {
+ let unit = normalized(turn.embedding)
+ let weight = Float(turn.end - turn.start)
+ if sum.isEmpty { sum = [Float](repeating: 0, count: unit.count) }
+ guard unit.count == sum.count else { continue }
+ for k in sum.indices { sum[k] += unit[k] * weight }
+ }
+ if !sum.isEmpty { centroids[speaker] = normalized(sum) }
+ }
+ let solidTurns = turns.filter { solid.contains($0.speaker) }
+ return turns.map { turn in
+ guard !solid.contains(turn.speaker) else { return turn }
+ var turn = turn
+ if !turn.embedding.isEmpty, centroids.count == solid.count {
+ let unit = normalized(turn.embedding)
+ if let best = centroids.max(by: { dot(unit, $0.value) < dot(unit, $1.value) }) {
+ turn.speaker = best.key
+ return turn
+ }
+ }
+ let mid = (turn.start + turn.end) / 2
+ if let nearest = solidTurns.min(by: { distance(mid, $0) < distance(mid, $1) }) {
+ turn.speaker = nearest.speaker
+ }
+ return turn
+ }
+ }
+
+ private static func normalized(_ v: [Float]) -> [Float] {
+ let norm = v.reduce(0) { $0 + $1 * $1 }.squareRoot()
+ return norm > 0 ? v.map { $0 / norm } : v
+ }
+
+ private static func dot(_ a: [Float], _ b: [Float]) -> Float {
+ guard a.count == b.count else { return -1 }
+ var sum: Float = 0
+ for i in a.indices { sum += a[i] * b[i] }
+ return sum
+ }
+
/// The whole transcript: your lines (echo removed) and the others', in time order, with the
/// speakers named in the order they first speak.
static func build(mine: [TimedWord], others: [TimedWord], turns: [SpeakerTurn],
@@ -287,3 +400,134 @@ enum NoteBuilder {
return Summary(title: title?.isEmpty == true ? nil : title, summary: summary, keyPoints: points, todos: todos)
}
}
+
+// MARK: - Correcting the transcript
+
+/// Speech recognition mishears names, brands and accented speech ("UG charging" for "EV charging").
+/// Claude reads the whole call and fixes those words; what it sends back is checked so it can only
+/// fix a line, never rewrite it (tests/NoteModelsTests.swift).
+extension NoteBuilder {
+ struct Correction: Sendable, Equatable {
+ /// Line number in the request → the line as it was said.
+ var lines: [Int: String]
+ /// Name shown in the transcript ("Person 2") → the person's name.
+ var speakers: [String: String]
+ }
+
+ /// Hesitations, left out when comparing a line with its correction.
+ static let fillers: Set<String> = ["uh", "um", "umm", "uhm", "er", "erm", "ah", "eh", "hmm", "mmm", "mm", "euh", "bah", "hum"]
+
+ /// Lines of the transcript cut in parts of about `words` words, for long calls.
+ static func batches(_ lines: [NoteLine], words: Int = 2_500) -> [Range<Int>] {
+ var out: [Range<Int>] = []
+ var start = 0, count = 0
+ for (index, line) in lines.enumerated() {
+ count += line.text.split(separator: " ").count
+ if count >= words {
+ out.append(start..<(index + 1))
+ start = index + 1
+ count = 0
+ }
+ }
+ if start < lines.count { out.append(start..<lines.count) }
+ return out
+ }
+
+ /// "[12] Person 2: words", one per line.
+ static func numberedTranscript(_ note: Note, range: Range<Int>) -> String {
+ range.map { index in
+ let line = note.lines[index]
+ return "[\(index)] \(note.speaker(line.speaker)?.name ?? line.speaker): \(line.text)"
+ }.joined(separator: "\n")
+ }
+
+ /// Reads {"lines": {"12": "…"}, "speakers": {"Person 2": "Pratap"}}, even with text around it.
+ static func parseCorrection(_ reply: String) -> Correction? {
+ guard let open = reply.firstIndex(of: "{"), let close = reply.lastIndex(of: "}"), open < close,
+ let data = String(reply[open...close]).data(using: .utf8),
+ let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil }
+ var lines: [Int: String] = [:]
+ for (key, value) in json["lines"] as? [String: Any] ?? [:] {
+ let digits = key.filter(\.isNumber)
+ guard let index = Int(digits), let text = value as? String else { continue }
+ let t = text.trimmingCharacters(in: .whitespacesAndNewlines)
+ if !t.isEmpty { lines[index] = t }
+ }
+ var speakers: [String: String] = [:]
+ for (key, value) in json["speakers"] as? [String: Any] ?? [:] {
+ guard let name = value as? String else { continue }
+ let n = name.trimmingCharacters(in: .whitespacesAndNewlines)
+ if !n.isEmpty { speakers[key] = n }
+ }
+ return Correction(lines: lines, speakers: speakers)
+ }
+
+ /// Whether a corrected line is still the same line: a few words fixed, hesitations gone,
+ /// not something else written instead.
+ static func plausibleFix(_ original: String, _ fixed: String) -> Bool {
+ let before = Array(tokens(original).filter { !fillers.contains($0) }.joined(separator: " "))
+ let after = Array(tokens(fixed).joined(separator: " "))
+ guard !after.isEmpty else { return false }
+ guard !before.isEmpty else { return after.count <= 12 }
+ let longest = max(before.count, after.count)
+ return Double(editDistance(before, after)) <= max(6, Double(longest) * 0.45)
+ }
+
+ static func editDistance(_ a: [Character], _ b: [Character]) -> Int {
+ if a.isEmpty { return b.count }
+ if b.isEmpty { return a.count }
+ var previous = Array(0...b.count)
+ var current = [Int](repeating: 0, count: b.count + 1)
+ for i in 1...a.count {
+ current[0] = i
+ for j in 1...b.count {
+ current[j] = a[i - 1] == b[j - 1] ? previous[j - 1] : 1 + min(previous[j - 1], previous[j], current[j - 1])
+ }
+ swap(&previous, &current)
+ }
+ return previous[b.count]
+ }
+
+ /// The note with the plausible fixes, and the people named where the call says who they are.
+ /// A speaker you renamed keeps your name; a name must be said somewhere in the call.
+ static func applying(_ correction: Correction, to note: Note, personName: (Int) -> String) -> (note: Note, fixed: Int) {
+ var note = note
+ var fixed = 0
+ for (index, text) in correction.lines where note.lines.indices.contains(index) {
+ let line = note.lines[index]
+ guard text != line.text, plausibleFix(line.text, text) else { continue }
+ if note.lines[index].raw == nil { note.lines[index].raw = line.text }
+ note.lines[index].text = text
+ fixed += 1
+ }
+ let spoken = note.lines.map { $0.raw ?? $0.text }.joined(separator: " ").lowercased()
+ + " " + note.lines.map(\.text).joined(separator: " ").lowercased()
+ for (shown, name) in correction.speakers {
+ guard name.count <= 40, name.split(separator: " ").count <= 3,
+ spoken.contains(name.lowercased()),
+ !note.speakers.contains(where: { $0.name.caseInsensitiveCompare(name) == .orderedSame }),
+ let index = note.speakers.firstIndex(where: { $0.name == shown && !$0.isMe }) else { continue }
+ let speaker = note.speakers[index]
+ // Only "Person 2" and the like: never a name you gave.
+ guard let number = Int(speaker.id.dropFirst()), speaker.name == personName(number) else { continue }
+ note.speakers[index].name = name
+ }
+ return (note, fixed)
+ }
+
+ /// The words a correction of yours brought in, worth remembering for the next calls: names,
+ /// brands, acronyms, technical words ("UG charging" → "EV charging" teaches "EV").
+ static func newTerms(old: String, new: String) -> [String] {
+ let before = Set(tokens(old))
+ var seen = Set<String>()
+ var out: [String] = []
+ for raw in new.split(whereSeparator: { $0.isWhitespace }) {
+ let word = raw.trimmingCharacters(in: CharacterSet.punctuationCharacters.subtracting(CharacterSet(charactersIn: "+#")))
+ let key = tokens(word).joined()
+ guard !key.isEmpty, !before.contains(key), !fillers.contains(key), seen.insert(key).inserted else { continue }
+ let distinctive = word.contains { $0.isUppercase || $0.isNumber } || word.contains(".") || word.count >= 7
+ if distinctive && word.count >= 2 { out.append(word) }
+ }
+ return out
+ }
+}
diff --git a/NotchBuddy/Sources/App/NoteTaker.swift b/NotchBuddy/Sources/App/NoteTaker.swift
index 582b1a26..5ce422d4 100644
--- a/NotchBuddy/Sources/App/NoteTaker.swift
+++ b/NotchBuddy/Sources/App/NoteTaker.swift
@@ -46,6 +46,10 @@ final class NoteTaker: ObservableObject {
/// Notes whose transcript or summary is being written.
@Published private(set) var working: Set<UUID> = []
+ /// Words and names learnt from your corrections (newest first): Claude gets them with every
+ /// transcript to correct, so a word fixed once is heard right in the next calls.
+ @Published private(set) var vocabulary: [String] = []
+
@Published var summaryEngine: NoteSummaryEngine {
didSet { UserDefaults.standard.set(summaryEngine.rawValue, forKey: Self.engineKey) }
}
@@ -72,6 +76,7 @@ final class NoteTaker: ObservableObject {
summaryEngine = FlowCleanup.hasKey ? .claude : (Self.appleModelAvailable ? .apple : .off)
}
load()
+ loadVocabulary()
appObserver = NSWorkspace.shared.notificationCenter.addObserver(
forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: .main
) { note in
@@ -127,8 +132,10 @@ final class NoteTaker: ObservableObject {
let dir = Self.folder(for: note.id)
do {
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
- let micChannel = try ChannelRecorder(url: dir.appendingPathComponent("mic.wav"), label: "fr.louisraille.NotchBuddy.notetaker.mic")
- let systemChannel = try ChannelRecorder(url: dir.appendingPathComponent("system.wav"), label: "fr.louisraille.NotchBuddy.notetaker.sys")
+ // One clock for both recordings, so the Mac's sound lines up with the microphone.
+ let origin = DispatchTime.now().uptimeNanoseconds
+ let micChannel = try ChannelRecorder(url: dir.appendingPathComponent("mic.wav"), label: "fr.louisraille.NotchBuddy.notetaker.mic", origin: origin)
+ let systemChannel = try ChannelRecorder(url: dir.appendingPathComponent("system.wav"), label: "fr.louisraille.NotchBuddy.notetaker.sys", origin: origin)
let mic = MicCapture()
guard mic.start(sink: { micChannel.append($0) }) else {
startError = String(localized: "notes.error.mic")
@@ -239,19 +246,26 @@ final class NoteTaker: ObservableObject {
throw NSError(domain: "Notetaker", code: 1, userInfo: [NSLocalizedDescriptionKey: String(localized: "notes.error.no-audio")])
}
let asr = try await Parakeet.shared.manager()
- let mine = try await Self.words(in: micURL, asr: asr)
+ var mine = try await Self.words(in: micURL, asr: asr)
var theirs: [TimedWord] = []
if FileManager.default.fileExists(atPath: systemURL.path) {
theirs = try await Self.words(in: systemURL, asr: asr)
}
let built: (speakers: [NoteSpeaker], lines: [NoteLine])
if theirs.count >= 5 {
- // A call: you on the microphone, the others on the Mac's sound.
- let turns = try await Self.diarize(systemURL)
+ // A call: you on the microphone, the others on the Mac's sound. Without headphones
+ // the others also come back into the microphone: that echo is taken out of your words.
+ let heard = mine, others = theirs
+ mine = await Task.detached(priority: .userInitiated) {
+ let byEar = NoteEcho.removing(heard, micLevels: Self.levels(micURL), sysLevels: Self.levels(systemURL))
+ return NoteBuilder.removingEchoWords(byEar.words, others: others)
+ }.value
+ appendAppLog("nb.log", "[Notetaker] echo: \(heard.count - mine.count) of \(heard.count) microphone words were the others")
+ let turns = NoteBuilder.mergingFragments(try await Self.diarize(systemURL))
built = NoteBuilder.build(mine: mine, others: theirs, turns: turns, meName: me, personName: Self.personName)
} else if mine.count >= 20 {
// People around the Mac: one recording, the voices told apart.
- let turns = try await Self.diarize(micURL)
+ let turns = NoteBuilder.mergingFragments(try await Self.diarize(micURL))
if Set(turns.map(\.speaker)).count > 1 {
built = NoteBuilder.buildRoom(words: mine, turns: turns, personName: Self.personName)
} else {
@@ -279,6 +293,9 @@ final class NoteTaker: ObservableObject {
replace(current)
return
}
+ if summaryEngine == .claude, !note.lines.isEmpty {
+ await correct(id)
+ }
if summaryEngine != .off, !note.lines.isEmpty {
await summarize(id)
}
@@ -319,8 +336,101 @@ final class NoteTaker: ObservableObject {
try await diarizer.prepareModels(directory: Parakeet.modelsRoot)
let result = try await diarizer.process(url)
return result.segments.map {
- SpeakerTurn(speaker: $0.speakerId, start: Double($0.startTimeSeconds), end: Double($0.endTimeSeconds))
+ SpeakerTurn(speaker: $0.speakerId, start: Double($0.startTimeSeconds), end: Double($0.endTimeSeconds),
+ embedding: $0.embedding)
+ }
+ }
+
+ /// The 20 ms levels of a recording, read a second at a time (a long call never sits in memory).
+ nonisolated private static func levels(_ url: URL) -> [Float] {
+ guard let file = try? AVAudioFile(forReading: url) else { return [] }
+ let chunk = AVAudioFrameCount(ChannelRecorder.sampleRate)
+ guard let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: chunk) else { return [] }
+ var out: [Float] = []
+ while file.framePosition < file.length {
+ guard (try? file.read(into: buffer, frameCount: chunk)) != nil, buffer.frameLength > 0,
+ let data = buffer.floatChannelData?[0] else { break }
+ out += NoteEcho.levels(Array(UnsafeBufferPointer(start: data, count: Int(buffer.frameLength))),
+ sampleRate: file.processingFormat.sampleRate)
+ }
+ return out
+ }
+
+ // MARK: - Correcting the transcript
+
+ nonisolated static let correctionInstructions = """
+ You fix the speech-recognition errors in the transcript of a call. The recognizer often mishears \
+ names, brands, technical terms, numbers, and accented or non-native speech: "UG charging" for \
+ "EV charging", "elemental codes" for "Elementor shortcodes".
+ Each line is "[n] Speaker: words". For each line that needs it, write it as the person actually said it:
+ - fix the misheard words, using the whole conversation, its topic and the vocabulary given;
+ - remove hesitations (uh, um, euh) and words repeated by accident (the the);
+ - keep the speaker's own words, grammar and language, even when imperfect or mixed: never rephrase, \
+ translate, shorten, complete or tidy up a sentence;
+ - never invent a name, number, address or link: when one is unclear, keep what was heard;
+ - when a passage stays impossible to understand, keep it and add [?] after it.
+ Also give the real name of a speaker when the call makes it certain (they say their own name, or \
+ are called by it); otherwise leave them out.
+ Reply ONLY with JSON, no code block, in this shape: \
+ {"lines": {"n": "the corrected words, without the speaker"}, "speakers": {"Speaker as written": "Name"}}
+ Put only the lines you changed.
+ """
+
+ /// Claude fixes the words the speech recognition got wrong (with the Claude key of Settings,
+ /// when Claude writes the summaries). What was heard stays in each line's `raw`.
+ func correct(_ id: UUID) async {
+ guard summaryEngine == .claude, let note = self.note(id), !note.lines.isEmpty else { return }
+ let key = KeychainStore.shared.get("anthropic-api-key") ?? ""
+ guard !key.isEmpty else { return }
+ working.insert(id)
+ defer { working.remove(id) }
+ let model = AppState.shared.claudeModel.isEmpty ? AppState.defaultClaudeModel : AppState.shared.claudeModel
+ var seen = Set<String>()
+ let terms = (vocabulary + FlowStore.shared.words).filter { seen.insert($0.lowercased()).inserted }.prefix(400)
+ var total = NoteBuilder.Correction(lines: [:], speakers: [:])
+ for range in NoteBuilder.batches(note.lines) {
+ var request = ""
+ if !terms.isEmpty { request += "<vocabulary>\n" + terms.joined(separator: ", ") + "\n</vocabulary>\n" }
+ request += "<transcript>\n" + NoteBuilder.numberedTranscript(note, range: range) + "\n</transcript>"
+ guard let reply = await Self.claude(system: Self.correctionInstructions, user: request, key: key,
+ model: model, maxTokens: 8_000),
+ let correction = NoteBuilder.parseCorrection(reply) else { continue }
+ total.lines.merge(correction.lines.filter { range.contains($0.key) }) { $1 }
+ total.speakers.merge(correction.speakers) { first, _ in first }
}
+ // Edited, reassigned or deleted meanwhile: only the lines still as they were are fixed.
+ guard let current = self.note(id), current.lines.map(\.id) == note.lines.map(\.id) else { return }
+ total.lines = total.lines.filter { current.lines[$0.key].text == note.lines[$0.key].text }
+ let result = NoteBuilder.applying(total, to: current, personName: Self.personName)
+ replace(result.note)
+ appendAppLog("nb.log", "[Notetaker] correction: \(result.fixed) of \(note.lines.count) lines fixed")
+ }
+
+ nonisolated private static func claude(system: String, user: String, key: String, model: String, maxTokens: Int) async -> String? {
+ func call(_ model: String) async -> String? {
+ var request = URLRequest(url: URL(string: "https://api.anthropic.com/v1/messages")!)
+ request.httpMethod = "POST"
+ request.timeoutInterval = 180
+ request.setValue(key, forHTTPHeaderField: "x-api-key")
+ request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")
+ request.setValue("application/json", forHTTPHeaderField: "content-type")
+ let body: [String: Any] = [
+ "model": model,
+ "max_tokens": maxTokens,
+ "system": system,
+ "messages": [["role": "user", "content": user]],
+ ]
+ guard let data = try? JSONSerialization.data(withJSONObject: body) else { return nil }
+ request.httpBody = data
+ guard let result = try? await URLSession.shared.data(for: request),
+ (result.1 as? HTTPURLResponse)?.statusCode == 200,
+ let json = try? JSONSerialization.jsonObject(with: result.0) as? [String: Any],
+ let content = json["content"] as? [[String: Any]] else { return nil }
+ let text = content.compactMap { $0["text"] as? String }.joined()
+ return text.isEmpty ? nil : text
+ }
+ if let text = await call(model) { return text }
+ return model == FlowCleanup.model ? nil : await call(FlowCleanup.model)
}
// MARK: - Summary
@@ -364,36 +474,16 @@ final class NoteTaker: ObservableObject {
nonisolated static let summaryInstructions = """
You write the notes of a meeting from its transcript. Each line is "Name: words".
Write in the language of the transcript. Invent nothing that was not said.
+ The transcript comes from speech recognition: a word that makes no sense was probably misheard, \
+ understand it from the context.
Reply ONLY with JSON, no code block, in this exact shape:
{"title": "a short title, 6 words at most", "summary": "2 to 4 sentences: what was decided and why", \
"key_points": ["up to 6 short points"], "todos": [{"text": "the task", "owner": "the person's name as written in the transcript, or null", "due": "the deadline as said, or null"}]}
"""
nonisolated private static func claudeSummary(_ transcript: String, key: String, model: String) async -> String? {
- func call(_ model: String) async -> String? {
- var request = URLRequest(url: URL(string: "https://api.anthropic.com/v1/messages")!)
- request.httpMethod = "POST"
- request.timeoutInterval = 120
- request.setValue(key, forHTTPHeaderField: "x-api-key")
- request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")
- request.setValue("application/json", forHTTPHeaderField: "content-type")
- let body: [String: Any] = [
- "model": model,
- "max_tokens": 2048,
- "system": summaryInstructions,
- "messages": [["role": "user", "content": "<transcript>\n\(transcript.prefix(400_000))\n</transcript>"]],
- ]
- guard let data = try? JSONSerialization.data(withJSONObject: body) else { return nil }
- request.httpBody = data
- guard let result = try? await URLSession.shared.data(for: request),
- (result.1 as? HTTPURLResponse)?.statusCode == 200,
- let json = try? JSONSerialization.jsonObject(with: result.0) as? [String: Any],
- let content = json["content"] as? [[String: Any]] else { return nil }
- let text = content.compactMap { $0["text"] as? String }.joined()
- return text.isEmpty ? nil : text
- }
- if let text = await call(model) { return text }
- return await call(FlowCleanup.model)
+ await claude(system: summaryInstructions, user: "<transcript>\n\(transcript.prefix(400_000))\n</transcript>",
+ key: key, model: model, maxTokens: 2048)
}
/// Apple's on-device model has a small context: a long call is first noted in parts, then the
@@ -438,14 +528,80 @@ final class NoteTaker: ObservableObject {
replace(note)
}
+ /// Renames someone. Given the name of another person of the call, the two are one: the diarizer
+ /// had cut one voice in two, and their lines are put together.
func rename(_ id: UUID, speaker: String, to name: String) {
guard var note = self.note(id), let index = note.speakers.firstIndex(where: { $0.id == speaker }) else { return }
let n = name.trimmingCharacters(in: .whitespacesAndNewlines)
guard !n.isEmpty else { return }
- note.speakers[index].name = n
+ let isMe = note.speakers[index].isMe
+ if !isMe, let other = note.speakers.first(where: { $0.id != speaker && !$0.isMe && $0.name.caseInsensitiveCompare(n) == .orderedSame }) {
+ for k in note.lines.indices where note.lines[k].speaker == speaker { note.lines[k].speaker = other.id }
+ note.speakers.remove(at: index)
+ } else {
+ note.speakers[index].name = n
+ }
+ if !isMe { learn([n]) }
+ replace(note)
+ }
+
+ /// You fix a line: the words you brought in are learnt for the next calls.
+ func edit(_ id: UUID, line: UUID, text: String) {
+ guard var note = self.note(id), let index = note.lines.firstIndex(where: { $0.id == line }) else { return }
+ let t = text.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !t.isEmpty, t != note.lines[index].text else { return }
+ learn(NoteBuilder.newTerms(old: note.lines[index].text, new: t))
+ if note.lines[index].raw == nil { note.lines[index].raw = note.lines[index].text }
+ note.lines[index].text = t
+ replace(note)
+ }
+
+ /// A line said by someone else than shown (an echo left, a voice taken for another).
+ func assign(_ id: UUID, line: UUID, to speaker: String) {
+ guard var note = self.note(id), note.speaker(speaker) != nil,
+ let index = note.lines.firstIndex(where: { $0.id == line }) else { return }
+ note.lines[index].speaker = speaker
+ replace(note)
+ }
+
+ func deleteLine(_ id: UUID, line: UUID) {
+ guard var note = self.note(id) else { return }
+ note.lines.removeAll { $0.id == line }
replace(note)
}
+ /// Correct the transcript again, then the summary from it.
+ func correctAgain(_ id: UUID) async {
+ await correct(id)
+ if summaryEngine != .off { await summarize(id) }
+ }
+
+ var canCorrect: Bool {
+ summaryEngine == .claude && FlowCleanup.hasKey
+ }
+
+ // MARK: - Vocabulary
+
+ private static var vocabularyURL: URL { folder.appendingPathComponent("vocabulary.json") }
+
+ func learn(_ terms: [String]) {
+ let fresh = terms.filter { term in !vocabulary.contains { $0.caseInsensitiveCompare(term) == .orderedSame } }
+ guard !fresh.isEmpty else { return }
+ vocabulary = Array((fresh + vocabulary).prefix(300))
+ do {
+ try FileManager.default.createDirectory(at: Self.folder, withIntermediateDirectories: true)
+ try JSONEncoder().encode(vocabulary).write(to: Self.vocabularyURL, options: .atomic)
+ } catch {
+ appendAppLog("nb.log", "[Notetaker] could not save the vocabulary: \(error.localizedDescription)")
+ }
+ }
+
+ private func loadVocabulary() {
+ guard let data = try? Data(contentsOf: Self.vocabularyURL),
+ let words = try? JSONDecoder().decode([String].self, from: data) else { return }
+ vocabulary = words
+ }
+
func toggle(_ id: UUID, todo: UUID) {
guard var note = self.note(id), let index = note.todos.firstIndex(where: { $0.id == todo }) else { return }
note.todos[index].done.toggle()
diff --git a/docs/SPEC.md b/docs/SPEC.md
index 70025fc4..3a84e049 100644
--- a/docs/SPEC.md
+++ b/docs/SPEC.md
@@ -439,3 +439,15 @@ Réglages → Raccourcis (`ShortcutsSettingsView`) :
- L'image arrive dans l'île (vue `.capture`) : son `gulp`, Mochi avale. Boutons : **Coller** (dans l'app où on était : ⌘V, ou ⌃V dans un terminal pour Claude Code ; le presse-papiers reprend son contenu après 1,5 s), **Copier**, **Annoter** (fenêtre : flèche, cadre, crayon, 4 couleurs, ⌘Z, l'image annotée revient dans l'île), **Demander à Claude** (chat de l'île, image jointe).
- Avec Flow : tant que la capture est dans l'île (3 min max), maintenir la touche de dictée colle l'image puis les mots au relâchement.
- Les captures restent dans le dossier temporaire (les 20 dernières) et ne partent vers Claude que si on les colle ou qu'on demande à Claude.
+
+## 18. Notetaker : écho, intervenants et mots mal entendus (build GitHub)
+
+Constaté sur un vrai appel sans casque : la voix de l'autre revenait dans le micro et passait pour la mienne, un même interlocuteur était coupé en deux personnes, et Parakeet écrivait « UG charging » pour « EV charging ».
+
+- **Une seule horloge.** Le micro et le son du Mac comptent leur temps depuis le même instant : le son du Mac, qui démarre un peu après, commence par le silence manquant. Les lignes et l'écho s'alignent.
+- **L'écho, au son.** Le son du Mac sert de référence : on mesure le délai et la force avec lesquels il revient dans le micro, puis un mot du micro qui n'a pas nettement plus d'énergie (≈ 10 dB) que cet écho est retiré. Avec un casque, l'écho est nul et rien n'est retiré ; parler par-dessus l'autre reste à moi.
+- **L'écho, aux mots.** En plus : trois mots de suite que l'autre a dits dans le même ordre au même moment, et la fin de sa phrase, sont retirés de mes lignes.
+- **Une voix coupée en deux.** Un « intervenant » qui ne parle que par bribes (jamais 3 s d'affilée, peu au total) est rendu à la voix la plus proche (empreinte vocale du diarizer, sinon la plus proche dans le temps).
+- **Mots mal entendus.** Quand Claude écrit les résumés (clé de Réglages → Chat), il corrige d'abord la transcription : mots mal reconnus, hésitations retirées, sans reformuler ni traduire, sans inventer de nom, de numéro ni de lien, `[?]` sur ce qui reste incompréhensible. Chaque correction est vérifiée (une ligne corrigée doit rester proche de l'originale) et ce qui avait été entendu est gardé (`raw`). Il nomme aussi un intervenant quand l'appel dit qui il est ; un nom donné par l'utilisateur n'est jamais remplacé.
+- **Au fil des appels.** Menu ⋯ d'une ligne (ou clic droit) : Corriger la ligne, Dit par…, Supprimer la ligne. Les mots apportés en corrigeant (noms, marques, sigles, mots techniques) et les noms donnés aux intervenants sont retenus (`Notes/vocabulary.json`, 300 max) et envoyés à Claude avec chaque transcription, avec le dictionnaire de Flow. Renommer un intervenant avec le nom d'un autre fusionne les deux.
+- Menu ⋯ de la note : « Corriger à nouveau la transcription ».
diff --git a/scripts/test-notetaker.sh b/scripts/test-notetaker.sh
index ed61427b..b0d2020c 100755
--- a/scripts/test-notetaker.sh
+++ b/scripts/test-notetaker.sh
@@ -3,5 +3,5 @@ set -euo pipefail
cd "$(dirname "$0")/.."
TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/coucou-notetaker.XXXXXX")"
trap 'rm -rf "$TEST_DIR"' EXIT
-swiftc NotchBuddy/Sources/App/NoteModels.swift tests/NoteModelsTests.swift -o "$TEST_DIR/notetaker-tests"
+swiftc NotchBuddy/Sources/App/NoteModels.swift NotchBuddy/Sources/App/NoteEcho.swift tests/NoteModelsTests.swift -o "$TEST_DIR/notetaker-tests"
"$TEST_DIR/notetaker-tests"
diff --git a/tests/NoteModelsTests.swift b/tests/NoteModelsTests.swift
index 5e86eeb9..2bdbf455 100644
--- a/tests/NoteModelsTests.swift
+++ b/tests/NoteModelsTests.swift
@@ -95,6 +95,116 @@ enum NoteModelsTests {
check("clock with hours", NoteBuilder.clock(3725) == "1:02:05")
}
+ print("\nEcho, word by word: their sentence inside one of my lines")
+ do {
+ // I talk, then their sentence comes back through the speakers, heard a little differently.
+ let mine = words("so what do you need", from: 0)
+ + words("currently we start with one page on this requirement", from: 10)
+ + words("okay perfect", from: 15)
+ let theirs = words("currently we start with one page for this particular category", from: 9.9)
+ let kept = NoteBuilder.removingEchoWords(mine, others: theirs)
+ check("my words stay", kept.map(\.text).joined(separator: " ") == "so what do you need okay perfect")
+ let lone = NoteBuilder.removingEchoWords(words("yeah sure", from: 3), others: words("yeah", from: 3))
+ check("two words in common are not an echo", lone.count == 2)
+ }
+
+ print("\nEcho, by the sound: the others' voice back in the microphone")
+ do {
+ let rate = 16_000.0
+ var seed: UInt32 = 7
+ func noise() -> Float { seed = seed &* 1_664_525 &+ 1_013_904_223; return Float(seed >> 8) / Float(1 << 24) - 0.5 }
+ // Syllables: 80–250 ms each, of random loudness, some silent.
+ var syllables: [Float] = []
+ while syllables.count < 200 { syllables.append(noise() < -0.3 ? 0.05 : 0.5 + noise()) }
+ var bounds: [Double] = [0]
+ while bounds.last! < 15 { bounds.append(bounds.last! + 0.165 + Double(noise()) * 0.17) }
+ func voice(_ t: Double, _ f: Double, _ amp: Float) -> Float {
+ let k = bounds.firstIndex { $0 > t } ?? 1
+ return Float(sin(2 * .pi * f * t)) * amp * syllables[(k + Int(f)) % syllables.count]
+ }
+ let count = Int(rate * 14)
+ var sys = [Float](repeating: 0, count: count), mic = [Float](repeating: 0, count: count)
+ let delay = Int(0.08 * rate)
+ for i in 0..<count {
+ let t = Double(i) / rate
+ // Them: 1–5 s and 9–11 s. Me: 6–7.5 s alone, and 9.3–10.2 s over them.
+ if (1..<5).contains(t) || (9..<11).contains(t) { sys[i] = voice(t, 180, 0.1) }
+ mic[i] = noise() * 0.002
+ if (6..<7.5).contains(t) || (9.3..<10.2).contains(t) { mic[i] += voice(t, 140, 0.12) }
+ }
+ for i in delay..<count { mic[i] += 0.2 * sys[i - delay] }
+ let heard = [TimedWord(text: "currently", start: 1.5, end: 2), TimedWord(text: "we", start: 2.1, end: 2.3),
+ TimedWord(text: "start", start: 2.4, end: 2.9),
+ TimedWord(text: "okay", start: 6.2, end: 6.6), TimedWord(text: "perfect", start: 6.8, end: 7.3),
+ TimedWord(text: "wait", start: 9.5, end: 9.9)]
+ let a = NoteEcho.analyze(mic: NoteEcho.levels(mic), sys: NoteEcho.levels(sys))
+ check("the delay is found (80 ms = 4 frames)", a.map { abs($0.lag - 4) <= 1 } == true)
+ check("the echo's strength is found", a.map { $0.gain > 0.1 && $0.gain < 0.4 } == true)
+ let result = NoteEcho.removing(heard, micSamples: mic, sysSamples: sys)
+ check("the echo is gone, my words stay, even over them",
+ result.words.map(\.text) == ["okay", "perfect", "wait"])
+ // With headphones: nothing comes back, nothing is removed.
+ var quiet = [Float](repeating: 0, count: count)
+ for i in 0..<count { quiet[i] = noise() * 0.002 + ((9.3..<10.2).contains(Double(i) / rate) ? voice(Double(i) / rate, 140, 0.12) : 0) }
+ let headphones = NoteEcho.removing([TimedWord(text: "wait", start: 9.5, end: 9.9)], micSamples: quiet, sysSamples: sys)
+ check("with headphones, my words stay", headphones.words.count == 1)
+ }
+
+ print("\nOne voice cut in two by the diarizer")
+ do {
+ let a: [Float] = [1, 0, 0.1], b: [Float] = [0, 1, 0]
+ let turns = [SpeakerTurn(speaker: "S1", start: 0, end: 20, embedding: a),
+ SpeakerTurn(speaker: "S2", start: 21, end: 21.8, embedding: [0.9, 0.1, 0.2]),
+ SpeakerTurn(speaker: "S3", start: 23, end: 40, embedding: b),
+ SpeakerTurn(speaker: "S2", start: 41, end: 41.5, embedding: [0.1, 0.8, 0])]
+ let merged = NoteBuilder.mergingFragments(turns)
+ check("the bits go to the closest voice", merged.map(\.speaker) == ["S1", "S1", "S3", "S3"])
+ let plain = NoteBuilder.mergingFragments([SpeakerTurn(speaker: "A", start: 0, end: 30),
+ SpeakerTurn(speaker: "B", start: 30.2, end: 31)])
+ check("without fingerprints, the nearest in time", plain.map(\.speaker) == ["A", "A"])
+ let two = NoteBuilder.mergingFragments([SpeakerTurn(speaker: "A", start: 0, end: 10),
+ SpeakerTurn(speaker: "B", start: 11, end: 20)])
+ check("two real voices stay two", Set(two.map(\.speaker)).count == 2)
+ }
+
+ print("\nCorrecting the transcript")
+ do {
+ var note = Note(title: "Call", start: Date())
+ note.speakers = [NoteSpeaker(id: "me", name: "Moi", color: 0, isMe: true),
+ NoteSpeaker(id: "p2", name: "Personne 2", color: 1)]
+ note.lines = [NoteLine(speaker: "p2", start: 0, end: 5, text: "uh we manufacture the EU charges primarily"),
+ NoteLine(speaker: "p2", start: 6, end: 9, text: "I built it with few other elemental codes right"),
+ NoteLine(speaker: "me", start: 10, end: 12, text: "okay perfect Pratap"),
+ NoteLine(speaker: "p2", start: 13, end: 15, text: "the price is fine")]
+ check("numbered", NoteBuilder.numberedTranscript(note, range: 0..<2).hasPrefix("[0] Personne 2: uh we"))
+ let reply = """
+ {"lines": {"0": "We manufacture the EV chargers primarily.", "[1]": "I built it with a few Elementor shortcodes, right.",
+ "3": "We will pay you one million dollars tomorrow morning."}, "speakers": {"Personne 2": "Pratap", "Moi": "Louis"}}
+ """
+ let correction = NoteBuilder.parseCorrection(reply)
+ check("parsed", correction?.lines.count == 3 && correction?.speakers["Personne 2"] == "Pratap")
+ let result = NoteBuilder.applying(correction!, to: note, personName: person)
+ check("misheard words fixed", result.note.lines[0].text == "We manufacture the EV chargers primarily."
+ && result.note.lines[1].text == "I built it with a few Elementor shortcodes, right.")
+ check("what was heard is kept", result.note.lines[0].raw == "uh we manufacture the EU charges primarily")
+ check("a rewritten line is refused", result.note.lines[3].text == "the price is fine" && result.fixed == 2)
+ check("the person is named, me stays me", result.note.speakers.map(\.name) == ["Moi", "Pratap"])
+ let invented = NoteBuilder.applying(.init(lines: [:], speakers: ["Personne 2": "Rahul"]), to: note, personName: person)
+ check("a name never said is not given", invented.note.speakers[1].name == "Personne 2")
+ var renamed = note
+ renamed.speakers[1].name = "Client"
+ check("your name is kept", NoteBuilder.applying(correction!, to: renamed, personName: person).note.speakers[1].name == "Client")
+ check("batches", NoteBuilder.batches(note.lines, words: 10) == [0..<2, 2..<4])
+ }
+
+ print("\nWords learnt from your corrections")
+ do {
+ check("acronyms and brands", NoteBuilder.newTerms(old: "the UG charging with elemental codes",
+ new: "the EV charging with Elementor shortcodes") == ["EV", "Elementor", "shortcodes"])
+ check("small common words are not learnt", NoteBuilder.newTerms(old: "it is fine", new: "it was fine").isEmpty)
+ check("a site", NoteBuilder.newTerms(old: "it's teslaws.com", new: "it's teslasvolt.com") == ["teslasvolt.com"])
+ }
+
if failures > 0 {
print("\n\(failures) failure(s)")
exit(1)