A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.
Swift ai-agentsanthropicantigravityclaudeclaude-codecodexcursordynamic-islandgemini-cliiosiphonlinuxmacosmacos-appmenubar-appnotchopen-sourceswiftswiftuiwindows
Notetaker: remove the echo by the sound, merge a voice cut in two, correct misheard words
Claudec6276017a69a
diff --git a/NotchBuddy/Resources/Localizable.xcstrings b/NotchBuddy/Resources/Localizable.xcstringsindex 1170e931..cf5d3197 100644--- a/NotchBuddy/Resources/Localizable.xcstrings+++ b/NotchBuddy/Resources/Localizable.xcstrings@@ -1,6 +1,396 @@{"sourceLanguage" : "en","strings" : {+ "notes.correct-again" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "تصحيح النص المكتوب من جديد"+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "ট্রান্সক্রিপ্ট আবার ঠিক করুন"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Correct the transcript again"+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corregir de nuevo la transcripción"+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corriger à nouveau la transcription"+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "ट्रांसक्रिप्ट फिर से सुधारें"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Koreksi transkrip lagi"+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corrigir a transcrição de novo"+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Исправить расшифровку заново"+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "重新校正转录"+ }+ }+ }+ },+ "notes.delete-line" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "حذف السطر"+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "লাইনটি মুছুন"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Delete the line"+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Eliminar la línea"+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Supprimer la ligne"+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "पंक्ति हटाएँ"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Hapus baris"+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Apagar a linha"+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Удалить строку"+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "删除这一行"+ }+ }+ }+ },+ "notes.edit-line" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "تصحيح السطر"+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "লাইনটি ঠিক করুন"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Fix the line"+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corregir la línea"+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corriger la ligne"+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "पंक्ति ठीक करें"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Perbaiki baris"+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Corrigir a linha"+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Исправить строку"+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "修改这一行"+ }+ }+ }+ },+ "notes.edit-line-detail" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "تُحفظ الكلمات التي تصححها لمكالماتك القادمة."+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "আপনি যে শব্দ ঠিক করেন, তা পরের কলগুলোর জন্য মনে রাখা হয়।"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "The words you bring in are remembered for your next calls."+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Las palabras que corriges se recuerdan para tus próximas llamadas."+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Les mots que tu corriges sont retenus pour tes prochains appels."+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "आप जो शब्द सुधारते हैं, वे अगली कॉल के लिए याद रखे जाते हैं।"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Kata yang kamu perbaiki diingat untuk panggilan berikutnya."+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "As palavras que você corrige ficam guardadas para as próximas chamadas."+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Исправленные слова запоминаются для следующих звонков."+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "你改正的词会被记住,用于之后的通话。"+ }+ }+ }+ },+ "notes.said-by" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "قالها"+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "কে বলেছেন"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Said by"+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Dicho por"+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Dit par"+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "किसने कहा"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Diucapkan oleh"+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Dito por"+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Сказал(а)"+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "说话人"+ }+ }+ }+ },+ "notes.transcript-hint" : {+ "extractionState" : "manual",+ "localizations" : {+ "ar" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "كلمة خاطئة أو شخص خاطئ؟ صحّحها من قائمة ⋯ في السطر. تُحفظ الكلمات التي تصححها لمكالماتك القادمة."+ }+ },+ "bn" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "ভুল শব্দ, নাকি ভুল ব্যক্তি? লাইনের ⋯ মেনু থেকে ঠিক করুন। আপনি যে শব্দ ঠিক করেন, তা পরের কলগুলোর জন্য মনে রাখা হয়।"+ }+ },+ "en" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "A wrong word, or the wrong person? Fix it from the line's ⋯ menu. Words you correct are remembered for your next calls."+ }+ },+ "es" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "¿Una palabra mal o la persona equivocada? Corrígelo desde el menú ⋯ de la línea. Las palabras que corriges se recuerdan para tus próximas llamadas."+ }+ },+ "fr" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Un mot faux, ou la mauvaise personne ? Corrige-le depuis le menu ⋯ de la ligne. Les mots que tu corriges sont retenus pour tes prochains appels."+ }+ },+ "hi" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "कोई शब्द गलत है, या व्यक्ति गलत? पंक्ति के ⋯ मेनू से ठीक करें। आप जो शब्द सुधारते हैं, वे अगली कॉल के लिए याद रखे जाते हैं।"+ }+ },+ "id" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Kata yang salah, atau orang yang salah? Perbaiki dari menu ⋯ di baris itu. Kata yang kamu perbaiki diingat untuk panggilan berikutnya."+ }+ },+ "pt-BR" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Uma palavra errada ou a pessoa errada? Corrija pelo menu ⋯ da linha. As palavras que você corrige ficam guardadas para as próximas chamadas."+ }+ },+ "ru" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "Неверное слово или не тот человек? Исправьте через меню ⋯ строки. Исправленные слова запоминаются для следующих звонков."+ }+ },+ "zh-Hans" : {+ "stringUnit" : {+ "state" : "translated",+ "value" : "词听错了,或说话人不对?在该行的 ⋯ 菜单里修改。你改正的词会被记住,用于之后的通话。"+ }+ }+ }+ },"vitals.title" : {"extractionState" : "manual","localizations" : {diff --git a/NotchBuddy/Sources/App/FlowHubNotes.swift b/NotchBuddy/Sources/App/FlowHubNotes.swiftindex 7bd71a57..584bec4a 100644--- a/NotchBuddy/Sources/App/FlowHubNotes.swift+++ b/NotchBuddy/Sources/App/FlowHubNotes.swift@@ -357,6 +357,8 @@ private struct HubNoteDetail: View {@State private var renaming: NoteSpeaker?@State private var newName = ""@State private var confirmDelete = false+ @State private var editing: NoteLine?+ @State private var editText = ""var body: some View {ScrollView {@@ -399,6 +401,16 @@ private struct HubNoteDetail: View {}Button(String(localized: "hub.cancel"), role: .cancel) { renaming = nil }}+ .alert(String(localized: "notes.edit-line"), isPresented: Binding(get: { editing != nil }, set: { if !$0 { editing = nil } })) {+ TextField("", text: $editText)+ Button(String(localized: "hub.save")) {+ if let line = editing { taker.edit(note.id, line: line.id, text: editText) }+ editing = nil+ }+ Button(String(localized: "hub.cancel"), role: .cancel) { editing = nil }+ } message: {+ Text("notes.edit-line-detail")+ }.alert(String(localized: "notes.delete-confirm"), isPresented: $confirmDelete) {Button(String(localized: "hub.delete"), role: .destructive) { taker.delete(note.id) }Button(String(localized: "hub.cancel"), role: .cancel) {}@@ -436,6 +448,8 @@ private struct HubNoteDetail: View {Button(String(localized: "notes.export")) { taker.export(note) }Button(String(localized: "notes.summarize-again")) { Task { await taker.summarize(note.id) } }.disabled(taker.summaryEngine == .off || note.lines.isEmpty)+ Button(String(localized: "notes.correct-again")) { Task { await taker.correctAgain(note.id) } }+ .disabled(!taker.canCorrect || note.lines.isEmpty || taker.working.contains(note.id))Divider()}Button(String(localized: "hub.delete"), role: .destructive) { confirmDelete = true }@@ -570,11 +584,64 @@ private struct HubNoteDetail: View {.font(.system(size: 13)).foregroundColor(HubTheme.secondary)}+ if !note.lines.isEmpty {+ Text("notes.transcript-hint")+ .font(.system(size: 12))+ .foregroundColor(HubTheme.muted)+ }ForEach(note.lines) { line inlet speaker = note.speaker(line.speaker)- HubLineView(name: speaker?.name ?? "?", color: HubTheme.speaker(speaker?.color ?? 1), line: line)+ HubEditableLine(note: note, line: line, name: speaker?.name ?? "?",+ color: HubTheme.speaker(speaker?.color ?? 1)) {+ editText = line.text+ editing = line+ }+ }+ }+ }+}++/// A line of a noted call, with what can be fixed on it: its words, who said it, or the line itself+/// (an echo left over). From the line's menu (shown on hover) or a right-click.+private struct HubEditableLine: View {+ let note: Note+ let line: NoteLine+ let name: String+ let color: Color+ let onEdit: () -> Void+ @ObservedObject private var taker = NoteTaker.shared+ @State private var hovering = false++ var body: some View {+ HubLineView(name: name, color: color, line: line)+ .frame(maxWidth: .infinity, alignment: .leading)+ .contentShape(Rectangle())+ .overlay(alignment: .topTrailing) {+ Menu { actions } label: {+ Image(systemName: "ellipsis")+ .font(.system(size: 12))+ .foregroundColor(HubTheme.secondary)+ }+ .menuStyle(.borderlessButton)+ .menuIndicator(.hidden)+ .fixedSize()+ .opacity(hovering ? 1 : 0)+ .help(String(localized: "notes.edit-line"))+ }+ .onHover { hovering = $0 }+ .contextMenu { actions }+ }++ @ViewBuilder private var actions: some View {+ Button(String(localized: "notes.edit-line"), action: onEdit)+ Menu(String(localized: "notes.said-by")) {+ ForEach(note.speakers) { other in+ Button(other.name) { taker.assign(note.id, line: line.id, to: other.id) }+ .disabled(other.id == line.speaker)}}+ Divider()+ Button(String(localized: "notes.delete-line"), role: .destructive) { taker.deleteLine(note.id, line: line.id) }}}diff --git a/NotchBuddy/Sources/App/NoteAudio.swift b/NotchBuddy/Sources/App/NoteAudio.swiftindex 0565ab71..89819bfb 100644--- a/NotchBuddy/Sources/App/NoteAudio.swift+++ b/NotchBuddy/Sources/App/NoteAudio.swift@@ -22,11 +22,16 @@ final class ChannelRecorder: @unchecked Sendable {var level: Float = 0/// Seconds with sound in them, to tell an empty side (no call, no headphones) from a real one.var voiced = 0.0+ var started = false}private let live = OSAllocatedUnfairLock(initialState: Live())- init(url: URL, label: String) throws {+ /// When the recording started (`DispatchTime` uptime): both sides count their time from it.+ private let origin: UInt64++ init(url: URL, label: String, origin: UInt64 = DispatchTime.now().uptimeNanoseconds) throws {self.url = url+ self.origin = originqueue = DispatchQueue(label: label, qos: .userInitiated)let settings: [String: Any] = [AVFormatIDKey: kAudioFormatLinearPCM,@@ -42,6 +47,19 @@ final class ChannelRecorder: @unchecked Sendable {/// From the audio thread.func append(_ samples: [Float]) {guard !samples.isEmpty else { return }+ let incoming = Double(samples.count) / Self.sampleRate+ let origin = self.origin+ // The Mac's sound starts a moment after the microphone (its device takes longer to open):+ // the silence before its first sound is written, so both recordings share one clock and+ // the echo and the order of the lines line up.+ let lead = live.withLock { state -> Int in+ guard !state.started else { return 0 }+ state.started = true+ let elapsed = Double(DispatchTime.now().uptimeNanoseconds &- origin) / 1_000_000_000+ let seconds = min(10, elapsed - incoming)+ return seconds > 0.01 ? Int(seconds * Self.sampleRate) : 0+ }+ let samples = lead > 0 ? [Float](repeating: 0, count: lead) + samples : sampleslet rms = Self.rms(samples[...])live.withLock { state instate.pending.append(contentsOf: samples)diff --git a/NotchBuddy/Sources/App/NoteEcho.swift b/NotchBuddy/Sources/App/NoteEcho.swiftnew file mode 100644index 00000000..41823d87--- /dev/null+++ b/NotchBuddy/Sources/App/NoteEcho.swift@@ -0,0 +1,145 @@+import Foundation++/// Without headphones, the others' voices come out of the speakers and back into the microphone,+/// and Parakeet writes them as yours (often heard a little differently, so comparing the words is+/// not enough). The Mac's own sound is the reference: where the microphone holds no more energy+/// than the echo of that sound explains, what it heard is the others, not you.+/// Pure functions on 16 kHz samples (tests/NoteModelsTests.swift).+enum NoteEcho {+ /// Levels are measured on 20 ms frames.+ static let frameSeconds = 0.02+ /// How much louder than the echo the microphone must be for a word to be yours (≈ 10 dB).+ static let margin: Float = 3+ /// The microphone lags behind what the Mac plays: searched from -100 ms to +500 ms.+ static let lags = -5...25++ struct Analysis: Sendable {+ /// Frames the microphone lags behind the Mac's sound.+ var lag: Int+ /// The microphone's level for one unit of the Mac's sound: the echo's strength (≈ 0 with headphones).+ var gain: Float+ /// Below this, the Mac plays nothing.+ var sysActive: Float+ /// The microphone's level in silence.+ var micFloor: Float+ var mic: [Float]+ var sys: [Float]+ }++ /// RMS level of each 20 ms frame.+ static func levels(_ samples: [Float], sampleRate: Double = 16_000) -> [Float] {+ let size = Int(sampleRate * frameSeconds)+ guard size > 0, !samples.isEmpty else { return [] }+ var out: [Float] = []+ out.reserveCapacity(samples.count / size + 1)+ var index = 0+ while index < samples.count {+ let end = min(samples.count, index + size)+ var sum: Float = 0+ for k in index..<end { sum += samples[k] * samples[k] }+ out.append((sum / Float(end - index)).squareRoot())+ index = end+ }+ return out+ }++ static func percentile(_ values: [Float], _ p: Double) -> Float {+ guard !values.isEmpty else { return 0 }+ let sorted = values.sorted()+ let index = min(sorted.count - 1, max(0, Int(Double(sorted.count - 1) * p)))+ return sorted[index]+ }++ /// How the Mac's sound reaches the microphone: the delay and the strength of the echo.+ /// nil when the Mac played too little sound to tell.+ static func analyze(mic: [Float], sys: [Float]) -> Analysis? {+ guard !mic.isEmpty, !sys.isEmpty else { return nil }+ let sysActive = max(0.004, percentile(sys, 0.1) * 4)+ let active = sys.indices.filter { sys[$0] > sysActive }+ guard active.count >= 50 else { return nil }+ let micFloor = percentile(mic, 0.1)++ // The delay: where the microphone's levels follow the Mac's best.+ var best = (lag: 0, corr: -Float.infinity)+ for lag in lags {+ var xs: [Float] = [], ys: [Float] = []+ for t in active where t + lag >= 0 && t + lag < mic.count {+ xs.append(sys[t])+ ys.append(mic[t + lag])+ }+ guard xs.count >= 50 else { continue }+ let corr = pearson(xs, ys)+ if corr > best.corr { best = (lag, corr) }+ }++ // The echo's strength: while only the others speak, the microphone is the Mac's sound times+ // this. When you speak over them the ratio is far higher, so a low percentile skips those moments.+ var ratios: [Float] = []+ for t in active where t + best.lag >= 0 && t + best.lag < mic.count {+ ratios.append(mic[t + best.lag] / sys[t])+ }+ guard ratios.count >= 50 else { return nil }+ return Analysis(lag: best.lag, gain: percentile(ratios, 0.35), sysActive: sysActive,+ micFloor: micFloor, mic: mic, sys: sys)+ }++ /// For one frame of the microphone: nil when the Mac plays nothing then, true when the+ /// microphone holds nothing more than the echo.+ static func isEcho(frame f: Int, in a: Analysis) -> Bool? {+ let t = f - a.lag+ guard f >= 0, f < a.mic.count, t >= 0, t < a.sys.count, a.sys[t] > a.sysActive else { return nil }+ return a.mic[f] <= margin * a.gain * a.sys[t] + 2 * a.micFloor+ }++ /// Which of the microphone's words are the echo of the others.+ static func echoWords(_ words: [TimedWord], analysis a: Analysis) -> [Bool] {+ var flags = words.map { word -> Bool in+ let first = max(0, Int(((word.start - 0.04) / frameSeconds).rounded(.down)))+ let last = max(first, Int(((word.end + 0.04) / frameSeconds).rounded(.up)))+ var active = 0, echo = 0+ for f in first...last {+ guard let isEcho = isEcho(frame: f, in: a) else { continue }+ active += 1+ if isEcho { echo += 1 }+ }+ let frames = last - first + 1+ return active * 2 >= frames && Double(echo) >= Double(active) * 0.6+ }+ // A lone word in the middle of the other kind, close to both neighbours, goes with them.+ let original = flags+ if words.count >= 3 {+ for i in 1..<(words.count - 1) where original[i - 1] == original[i + 1] && original[i] != original[i - 1] {+ if words[i].start - words[i - 1].end < 0.6, words[i + 1].start - words[i].end < 0.6 {+ flags[i] = original[i - 1]+ }+ }+ }+ return flags+ }++ /// The microphone's words without the echo, from both recordings' levels.+ static func removing(_ words: [TimedWord], micLevels: [Float], sysLevels: [Float]) -> (words: [TimedWord], removed: Int) {+ guard let analysis = analyze(mic: micLevels, sys: sysLevels) else { return (words, 0) }+ let flags = echoWords(words, analysis: analysis)+ let kept = zip(words, flags).filter { !$0.1 }.map(\.0)+ return (kept, words.count - kept.count)+ }++ static func removing(_ words: [TimedWord], micSamples: [Float], sysSamples: [Float]) -> (words: [TimedWord], removed: Int) {+ removing(words, micLevels: levels(micSamples), sysLevels: levels(sysSamples))+ }++ private static func pearson(_ xs: [Float], _ ys: [Float]) -> Float {+ let n = Float(xs.count)+ let mx = xs.reduce(0, +) / n, my = ys.reduce(0, +) / n+ var sxy: Float = 0, sxx: Float = 0, syy: Float = 0+ for i in xs.indices {+ let dx = xs[i] - mx, dy = ys[i] - my+ sxy += dx * dy+ sxx += dx * dx+ syy += dy * dy+ }+ guard sxx > 0, syy > 0 else { return 0 }+ return sxy / (sxx * syy).squareRoot()+ }+}diff --git a/NotchBuddy/Sources/App/NoteModels.swift b/NotchBuddy/Sources/App/NoteModels.swiftindex 02ff20d2..9e5d5f7a 100644--- a/NotchBuddy/Sources/App/NoteModels.swift+++ b/NotchBuddy/Sources/App/NoteModels.swift@@ -17,6 +17,8 @@ struct NoteLine: Codable, Hashable, Identifiable, Sendable {var start: Doublevar end: Doublevar text: String+ /// The words as Parakeet heard them, kept when the line was corrected (by Claude or by you).+ var raw: String? = nil}struct NoteTodo: Codable, Hashable, Identifiable, Sendable {@@ -75,6 +77,8 @@ struct SpeakerTurn: Hashable, Sendable {var speaker: Stringvar start: Doublevar end: Double+ /// The voice's fingerprint on this stretch (empty when unknown).+ var embedding: [Float] = []}/// Builds the transcript of a call from the two recordings (tests/NoteModelsTests.swift):@@ -145,6 +149,115 @@ enum NoteBuilder {s.lowercased().split { !($0.isLetter || $0.isNumber) }.map(String.init)}+ /// The echo found word by word: three or more of your words in a row that the others said in+ /// the same order at the same moment (one word heard differently, or missed, does not break it).+ /// Catches what the audio check lets through, when the speakers are very loud.+ static func removingEchoWords(_ mine: [TimedWord], others: [TimedWord],+ window: Double = 1.2, minRun: Int = 3) -> [TimedWord] {+ let a = mine.map { tokens($0.text).joined() }+ let b = others.map { tokens($0.text).joined() }+ var echo = [Bool](repeating: false, count: mine.count)+ var i = 0+ while i < mine.count {+ var span = 0+ if !a[i].isEmpty {+ for j in others.indices where b[j] == a[i] && abs(others[j].start - mine[i].start) <= window {+ var x = i + 1, y = j + 1, run = 1+ while x < mine.count && y < others.count {+ if a[x] == b[y] {+ run += 1; x += 1; y += 1+ } else if y + 1 < others.count && a[x] == b[y + 1] {+ y += 1 // a word the microphone missed+ } else if x + 1 < mine.count && a[x + 1] == b[y] {+ x += 1 // a word the microphone heard in more+ } else if x + 1 < mine.count && y + 1 < others.count && a[x + 1] == b[y + 1] {+ x += 1; y += 1 // a word heard differently+ } else {+ break+ }+ }+ guard run >= minRun else { continue }+ // The rest of their sentence, heard too differently to match, is the echo as well:+ // your words that follow without a pause while they are still speaking.+ var k = max(j, y - 1)+ while k + 1 < others.count && others[k + 1].start - others[k].end < 0.6 { k += 1 }+ let until = others[min(k, others.count - 1)].end + 0.3+ while x < mine.count && mine[x].start <= until && mine[x].start - mine[x - 1].end < 0.6 { x += 1 }+ span = max(span, x - i)+ }+ }+ if span > 0 {+ for k in i..<(i + span) { echo[k] = true }+ i += span+ } else {+ i += 1+ }+ }+ return zip(mine, echo).filter { !$0.1 }.map(\.0)+ }++ /// A voice heard only in short bits ("Yeah.", "Sure.", the end of a sentence) is almost always+ /// someone already there: such bits are too short for the diarizer to recognise the voice, and+ /// it makes a new person of them. Each bit goes to the closest real voice (by fingerprint,+ /// else the nearest in time). A voice is real once it speaks 3 s in a row, or long enough overall.+ static let fragmentLongest = 3.0++ static func mergingFragments(_ turns: [SpeakerTurn]) -> [SpeakerTurn] {+ var total: [String: Double] = [:], longest: [String: Double] = [:]+ for turn in turns {+ let length = turn.end - turn.start+ total[turn.speaker, default: 0] += length+ longest[turn.speaker] = max(longest[turn.speaker] ?? 0, length)+ }+ guard total.count > 1 else { return turns }+ let all = total.values.reduce(0, +)+ let solid = Set(total.keys.filter { (longest[$0] ?? 0) >= fragmentLongest || (total[$0] ?? 0) >= max(20, all * 0.1) })+ guard !solid.isEmpty, solid.count < total.count else { return turns }++ // Each real voice's fingerprint: the mean of its turns', longer turns counting more.+ var centroids: [String: [Float]] = [:]+ for speaker in solid {+ var sum: [Float] = []+ for turn in turns where turn.speaker == speaker && !turn.embedding.isEmpty {+ let unit = normalized(turn.embedding)+ let weight = Float(turn.end - turn.start)+ if sum.isEmpty { sum = [Float](repeating: 0, count: unit.count) }+ guard unit.count == sum.count else { continue }+ for k in sum.indices { sum[k] += unit[k] * weight }+ }+ if !sum.isEmpty { centroids[speaker] = normalized(sum) }+ }+ let solidTurns = turns.filter { solid.contains($0.speaker) }+ return turns.map { turn in+ guard !solid.contains(turn.speaker) else { return turn }+ var turn = turn+ if !turn.embedding.isEmpty, centroids.count == solid.count {+ let unit = normalized(turn.embedding)+ if let best = centroids.max(by: { dot(unit, $0.value) < dot(unit, $1.value) }) {+ turn.speaker = best.key+ return turn+ }+ }+ let mid = (turn.start + turn.end) / 2+ if let nearest = solidTurns.min(by: { distance(mid, $0) < distance(mid, $1) }) {+ turn.speaker = nearest.speaker+ }+ return turn+ }+ }++ private static func normalized(_ v: [Float]) -> [Float] {+ let norm = v.reduce(0) { $0 + $1 * $1 }.squareRoot()+ return norm > 0 ? v.map { $0 / norm } : v+ }++ private static func dot(_ a: [Float], _ b: [Float]) -> Float {+ guard a.count == b.count else { return -1 }+ var sum: Float = 0+ for i in a.indices { sum += a[i] * b[i] }+ return sum+ }+/// The whole transcript: your lines (echo removed) and the others', in time order, with the/// speakers named in the order they first speak.static func build(mine: [TimedWord], others: [TimedWord], turns: [SpeakerTurn],@@ -287,3 +400,134 @@ enum NoteBuilder {return Summary(title: title?.isEmpty == true ? nil : title, summary: summary, keyPoints: points, todos: todos)}}++// MARK: - Correcting the transcript++/// Speech recognition mishears names, brands and accented speech ("UG charging" for "EV charging").+/// Claude reads the whole call and fixes those words; what it sends back is checked so it can only+/// fix a line, never rewrite it (tests/NoteModelsTests.swift).+extension NoteBuilder {+ struct Correction: Sendable, Equatable {+ /// Line number in the request → the line as it was said.+ var lines: [Int: String]+ /// Name shown in the transcript ("Person 2") → the person's name.+ var speakers: [String: String]+ }++ /// Hesitations, left out when comparing a line with its correction.+ static let fillers: Set<String> = ["uh", "um", "umm", "uhm", "er", "erm", "ah", "eh", "hmm", "mmm", "mm", "euh", "bah", "hum"]++ /// Lines of the transcript cut in parts of about `words` words, for long calls.+ static func batches(_ lines: [NoteLine], words: Int = 2_500) -> [Range<Int>] {+ var out: [Range<Int>] = []+ var start = 0, count = 0+ for (index, line) in lines.enumerated() {+ count += line.text.split(separator: " ").count+ if count >= words {+ out.append(start..<(index + 1))+ start = index + 1+ count = 0+ }+ }+ if start < lines.count { out.append(start..<lines.count) }+ return out+ }++ /// "[12] Person 2: words", one per line.+ static func numberedTranscript(_ note: Note, range: Range<Int>) -> String {+ range.map { index in+ let line = note.lines[index]+ return "[\(index)] \(note.speaker(line.speaker)?.name ?? line.speaker): \(line.text)"+ }.joined(separator: "\n")+ }++ /// Reads {"lines": {"12": "…"}, "speakers": {"Person 2": "Pratap"}}, even with text around it.+ static func parseCorrection(_ reply: String) -> Correction? {+ guard let open = reply.firstIndex(of: "{"), let close = reply.lastIndex(of: "}"), open < close,+ let data = String(reply[open...close]).data(using: .utf8),+ let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil }+ var lines: [Int: String] = [:]+ for (key, value) in json["lines"] as? [String: Any] ?? [:] {+ let digits = key.filter(\.isNumber)+ guard let index = Int(digits), let text = value as? String else { continue }+ let t = text.trimmingCharacters(in: .whitespacesAndNewlines)+ if !t.isEmpty { lines[index] = t }+ }+ var speakers: [String: String] = [:]+ for (key, value) in json["speakers"] as? [String: Any] ?? [:] {+ guard let name = value as? String else { continue }+ let n = name.trimmingCharacters(in: .whitespacesAndNewlines)+ if !n.isEmpty { speakers[key] = n }+ }+ return Correction(lines: lines, speakers: speakers)+ }++ /// Whether a corrected line is still the same line: a few words fixed, hesitations gone,+ /// not something else written instead.+ static func plausibleFix(_ original: String, _ fixed: String) -> Bool {+ let before = Array(tokens(original).filter { !fillers.contains($0) }.joined(separator: " "))+ let after = Array(tokens(fixed).joined(separator: " "))+ guard !after.isEmpty else { return false }+ guard !before.isEmpty else { return after.count <= 12 }+ let longest = max(before.count, after.count)+ return Double(editDistance(before, after)) <= max(6, Double(longest) * 0.45)+ }++ static func editDistance(_ a: [Character], _ b: [Character]) -> Int {+ if a.isEmpty { return b.count }+ if b.isEmpty { return a.count }+ var previous = Array(0...b.count)+ var current = [Int](repeating: 0, count: b.count + 1)+ for i in 1...a.count {+ current[0] = i+ for j in 1...b.count {+ current[j] = a[i - 1] == b[j - 1] ? previous[j - 1] : 1 + min(previous[j - 1], previous[j], current[j - 1])+ }+ swap(&previous, ¤t)+ }+ return previous[b.count]+ }++ /// The note with the plausible fixes, and the people named where the call says who they are.+ /// A speaker you renamed keeps your name; a name must be said somewhere in the call.+ static func applying(_ correction: Correction, to note: Note, personName: (Int) -> String) -> (note: Note, fixed: Int) {+ var note = note+ var fixed = 0+ for (index, text) in correction.lines where note.lines.indices.contains(index) {+ let line = note.lines[index]+ guard text != line.text, plausibleFix(line.text, text) else { continue }+ if note.lines[index].raw == nil { note.lines[index].raw = line.text }+ note.lines[index].text = text+ fixed += 1+ }+ let spoken = note.lines.map { $0.raw ?? $0.text }.joined(separator: " ").lowercased()+ + " " + note.lines.map(\.text).joined(separator: " ").lowercased()+ for (shown, name) in correction.speakers {+ guard name.count <= 40, name.split(separator: " ").count <= 3,+ spoken.contains(name.lowercased()),+ !note.speakers.contains(where: { $0.name.caseInsensitiveCompare(name) == .orderedSame }),+ let index = note.speakers.firstIndex(where: { $0.name == shown && !$0.isMe }) else { continue }+ let speaker = note.speakers[index]+ // Only "Person 2" and the like: never a name you gave.+ guard let number = Int(speaker.id.dropFirst()), speaker.name == personName(number) else { continue }+ note.speakers[index].name = name+ }+ return (note, fixed)+ }++ /// The words a correction of yours brought in, worth remembering for the next calls: names,+ /// brands, acronyms, technical words ("UG charging" → "EV charging" teaches "EV").+ static func newTerms(old: String, new: String) -> [String] {+ let before = Set(tokens(old))+ var seen = Set<String>()+ var out: [String] = []+ for raw in new.split(whereSeparator: { $0.isWhitespace }) {+ let word = raw.trimmingCharacters(in: CharacterSet.punctuationCharacters.subtracting(CharacterSet(charactersIn: "+#")))+ let key = tokens(word).joined()+ guard !key.isEmpty, !before.contains(key), !fillers.contains(key), seen.insert(key).inserted else { continue }+ let distinctive = word.contains { $0.isUppercase || $0.isNumber } || word.contains(".") || word.count >= 7+ if distinctive && word.count >= 2 { out.append(word) }+ }+ return out+ }+}diff --git a/NotchBuddy/Sources/App/NoteTaker.swift b/NotchBuddy/Sources/App/NoteTaker.swiftindex 582b1a26..5ce422d4 100644--- a/NotchBuddy/Sources/App/NoteTaker.swift+++ b/NotchBuddy/Sources/App/NoteTaker.swift@@ -46,6 +46,10 @@ final class NoteTaker: ObservableObject {/// Notes whose transcript or summary is being written.@Published private(set) var working: Set<UUID> = []+ /// Words and names learnt from your corrections (newest first): Claude gets them with every+ /// transcript to correct, so a word fixed once is heard right in the next calls.+ @Published private(set) var vocabulary: [String] = []+@Published var summaryEngine: NoteSummaryEngine {didSet { UserDefaults.standard.set(summaryEngine.rawValue, forKey: Self.engineKey) }}@@ -72,6 +76,7 @@ final class NoteTaker: ObservableObject {summaryEngine = FlowCleanup.hasKey ? .claude : (Self.appleModelAvailable ? .apple : .off)}load()+ loadVocabulary()appObserver = NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: .main) { note in@@ -127,8 +132,10 @@ final class NoteTaker: ObservableObject {let dir = Self.folder(for: note.id)do {try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)- let micChannel = try ChannelRecorder(url: dir.appendingPathComponent("mic.wav"), label: "fr.louisraille.NotchBuddy.notetaker.mic")- let systemChannel = try ChannelRecorder(url: dir.appendingPathComponent("system.wav"), label: "fr.louisraille.NotchBuddy.notetaker.sys")+ // One clock for both recordings, so the Mac's sound lines up with the microphone.+ let origin = DispatchTime.now().uptimeNanoseconds+ let micChannel = try ChannelRecorder(url: dir.appendingPathComponent("mic.wav"), label: "fr.louisraille.NotchBuddy.notetaker.mic", origin: origin)+ let systemChannel = try ChannelRecorder(url: dir.appendingPathComponent("system.wav"), label: "fr.louisraille.NotchBuddy.notetaker.sys", origin: origin)let mic = MicCapture()guard mic.start(sink: { micChannel.append($0) }) else {startError = String(localized: "notes.error.mic")@@ -239,19 +246,26 @@ final class NoteTaker: ObservableObject {throw NSError(domain: "Notetaker", code: 1, userInfo: [NSLocalizedDescriptionKey: String(localized: "notes.error.no-audio")])}let asr = try await Parakeet.shared.manager()- let mine = try await Self.words(in: micURL, asr: asr)+ var mine = try await Self.words(in: micURL, asr: asr)var theirs: [TimedWord] = []if FileManager.default.fileExists(atPath: systemURL.path) {theirs = try await Self.words(in: systemURL, asr: asr)}let built: (speakers: [NoteSpeaker], lines: [NoteLine])if theirs.count >= 5 {- // A call: you on the microphone, the others on the Mac's sound.- let turns = try await Self.diarize(systemURL)+ // A call: you on the microphone, the others on the Mac's sound. Without headphones+ // the others also come back into the microphone: that echo is taken out of your words.+ let heard = mine, others = theirs+ mine = await Task.detached(priority: .userInitiated) {+ let byEar = NoteEcho.removing(heard, micLevels: Self.levels(micURL), sysLevels: Self.levels(systemURL))+ return NoteBuilder.removingEchoWords(byEar.words, others: others)+ }.value+ appendAppLog("nb.log", "[Notetaker] echo: \(heard.count - mine.count) of \(heard.count) microphone words were the others")+ let turns = NoteBuilder.mergingFragments(try await Self.diarize(systemURL))built = NoteBuilder.build(mine: mine, others: theirs, turns: turns, meName: me, personName: Self.personName)} else if mine.count >= 20 {// People around the Mac: one recording, the voices told apart.- let turns = try await Self.diarize(micURL)+ let turns = NoteBuilder.mergingFragments(try await Self.diarize(micURL))if Set(turns.map(\.speaker)).count > 1 {built = NoteBuilder.buildRoom(words: mine, turns: turns, personName: Self.personName)} else {@@ -279,6 +293,9 @@ final class NoteTaker: ObservableObject {replace(current)return}+ if summaryEngine == .claude, !note.lines.isEmpty {+ await correct(id)+ }if summaryEngine != .off, !note.lines.isEmpty {await summarize(id)}@@ -319,8 +336,101 @@ final class NoteTaker: ObservableObject {try await diarizer.prepareModels(directory: Parakeet.modelsRoot)let result = try await diarizer.process(url)return result.segments.map {- SpeakerTurn(speaker: $0.speakerId, start: Double($0.startTimeSeconds), end: Double($0.endTimeSeconds))+ SpeakerTurn(speaker: $0.speakerId, start: Double($0.startTimeSeconds), end: Double($0.endTimeSeconds),+ embedding: $0.embedding)+ }+ }++ /// The 20 ms levels of a recording, read a second at a time (a long call never sits in memory).+ nonisolated private static func levels(_ url: URL) -> [Float] {+ guard let file = try? AVAudioFile(forReading: url) else { return [] }+ let chunk = AVAudioFrameCount(ChannelRecorder.sampleRate)+ guard let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: chunk) else { return [] }+ var out: [Float] = []+ while file.framePosition < file.length {+ guard (try? file.read(into: buffer, frameCount: chunk)) != nil, buffer.frameLength > 0,+ let data = buffer.floatChannelData?[0] else { break }+ out += NoteEcho.levels(Array(UnsafeBufferPointer(start: data, count: Int(buffer.frameLength))),+ sampleRate: file.processingFormat.sampleRate)+ }+ return out+ }++ // MARK: - Correcting the transcript++ nonisolated static let correctionInstructions = """+ You fix the speech-recognition errors in the transcript of a call. The recognizer often mishears \+ names, brands, technical terms, numbers, and accented or non-native speech: "UG charging" for \+ "EV charging", "elemental codes" for "Elementor shortcodes".+ Each line is "[n] Speaker: words". For each line that needs it, write it as the person actually said it:+ - fix the misheard words, using the whole conversation, its topic and the vocabulary given;+ - remove hesitations (uh, um, euh) and words repeated by accident (the the);+ - keep the speaker's own words, grammar and language, even when imperfect or mixed: never rephrase, \+ translate, shorten, complete or tidy up a sentence;+ - never invent a name, number, address or link: when one is unclear, keep what was heard;+ - when a passage stays impossible to understand, keep it and add [?] after it.+ Also give the real name of a speaker when the call makes it certain (they say their own name, or \+ are called by it); otherwise leave them out.+ Reply ONLY with JSON, no code block, in this shape: \+ {"lines": {"n": "the corrected words, without the speaker"}, "speakers": {"Speaker as written": "Name"}}+ Put only the lines you changed.+ """++ /// Claude fixes the words the speech recognition got wrong (with the Claude key of Settings,+ /// when Claude writes the summaries). What was heard stays in each line's `raw`.+ func correct(_ id: UUID) async {+ guard summaryEngine == .claude, let note = self.note(id), !note.lines.isEmpty else { return }+ let key = KeychainStore.shared.get("anthropic-api-key") ?? ""+ guard !key.isEmpty else { return }+ working.insert(id)+ defer { working.remove(id) }+ let model = AppState.shared.claudeModel.isEmpty ? AppState.defaultClaudeModel : AppState.shared.claudeModel+ var seen = Set<String>()+ let terms = (vocabulary + FlowStore.shared.words).filter { seen.insert($0.lowercased()).inserted }.prefix(400)+ var total = NoteBuilder.Correction(lines: [:], speakers: [:])+ for range in NoteBuilder.batches(note.lines) {+ var request = ""+ if !terms.isEmpty { request += "<vocabulary>\n" + terms.joined(separator: ", ") + "\n</vocabulary>\n" }+ request += "<transcript>\n" + NoteBuilder.numberedTranscript(note, range: range) + "\n</transcript>"+ guard let reply = await Self.claude(system: Self.correctionInstructions, user: request, key: key,+ model: model, maxTokens: 8_000),+ let correction = NoteBuilder.parseCorrection(reply) else { continue }+ total.lines.merge(correction.lines.filter { range.contains($0.key) }) { $1 }+ total.speakers.merge(correction.speakers) { first, _ in first }}+ // Edited, reassigned or deleted meanwhile: only the lines still as they were are fixed.+ guard let current = self.note(id), current.lines.map(\.id) == note.lines.map(\.id) else { return }+ total.lines = total.lines.filter { current.lines[$0.key].text == note.lines[$0.key].text }+ let result = NoteBuilder.applying(total, to: current, personName: Self.personName)+ replace(result.note)+ appendAppLog("nb.log", "[Notetaker] correction: \(result.fixed) of \(note.lines.count) lines fixed")+ }++ nonisolated private static func claude(system: String, user: String, key: String, model: String, maxTokens: Int) async -> String? {+ func call(_ model: String) async -> String? {+ var request = URLRequest(url: URL(string: "https://api.anthropic.com/v1/messages")!)+ request.httpMethod = "POST"+ request.timeoutInterval = 180+ request.setValue(key, forHTTPHeaderField: "x-api-key")+ request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")+ request.setValue("application/json", forHTTPHeaderField: "content-type")+ let body: [String: Any] = [+ "model": model,+ "max_tokens": maxTokens,+ "system": system,+ "messages": [["role": "user", "content": user]],+ ]+ guard let data = try? JSONSerialization.data(withJSONObject: body) else { return nil }+ request.httpBody = data+ guard let result = try? await URLSession.shared.data(for: request),+ (result.1 as? HTTPURLResponse)?.statusCode == 200,+ let json = try? JSONSerialization.jsonObject(with: result.0) as? [String: Any],+ let content = json["content"] as? [[String: Any]] else { return nil }+ let text = content.compactMap { $0["text"] as? String }.joined()+ return text.isEmpty ? nil : text+ }+ if let text = await call(model) { return text }+ return model == FlowCleanup.model ? nil : await call(FlowCleanup.model)}// MARK: - Summary@@ -364,36 +474,16 @@ final class NoteTaker: ObservableObject {nonisolated static let summaryInstructions = """You write the notes of a meeting from its transcript. Each line is "Name: words".Write in the language of the transcript. Invent nothing that was not said.+ The transcript comes from speech recognition: a word that makes no sense was probably misheard, \+ understand it from the context.Reply ONLY with JSON, no code block, in this exact shape:{"title": "a short title, 6 words at most", "summary": "2 to 4 sentences: what was decided and why", \"key_points": ["up to 6 short points"], "todos": [{"text": "the task", "owner": "the person's name as written in the transcript, or null", "due": "the deadline as said, or null"}]}"""nonisolated private static func claudeSummary(_ transcript: String, key: String, model: String) async -> String? {- func call(_ model: String) async -> String? {- var request = URLRequest(url: URL(string: "https://api.anthropic.com/v1/messages")!)- request.httpMethod = "POST"- request.timeoutInterval = 120- request.setValue(key, forHTTPHeaderField: "x-api-key")- request.setValue("2023-06-01", forHTTPHeaderField: "anthropic-version")- request.setValue("application/json", forHTTPHeaderField: "content-type")- let body: [String: Any] = [- "model": model,- "max_tokens": 2048,- "system": summaryInstructions,- "messages": [["role": "user", "content": "<transcript>\n\(transcript.prefix(400_000))\n</transcript>"]],- ]- guard let data = try? JSONSerialization.data(withJSONObject: body) else { return nil }- request.httpBody = data- guard let result = try? await URLSession.shared.data(for: request),- (result.1 as? HTTPURLResponse)?.statusCode == 200,- let json = try? JSONSerialization.jsonObject(with: result.0) as? [String: Any],- let content = json["content"] as? [[String: Any]] else { return nil }- let text = content.compactMap { $0["text"] as? String }.joined()- return text.isEmpty ? nil : text- }- if let text = await call(model) { return text }- return await call(FlowCleanup.model)+ await claude(system: summaryInstructions, user: "<transcript>\n\(transcript.prefix(400_000))\n</transcript>",+ key: key, model: model, maxTokens: 2048)}/// Apple's on-device model has a small context: a long call is first noted in parts, then the@@ -438,14 +528,80 @@ final class NoteTaker: ObservableObject {replace(note)}+ /// Renames someone. Given the name of another person of the call, the two are one: the diarizer+ /// had cut one voice in two, and their lines are put together.func rename(_ id: UUID, speaker: String, to name: String) {guard var note = self.note(id), let index = note.speakers.firstIndex(where: { $0.id == speaker }) else { return }let n = name.trimmingCharacters(in: .whitespacesAndNewlines)guard !n.isEmpty else { return }- note.speakers[index].name = n+ let isMe = note.speakers[index].isMe+ if !isMe, let other = note.speakers.first(where: { $0.id != speaker && !$0.isMe && $0.name.caseInsensitiveCompare(n) == .orderedSame }) {+ for k in note.lines.indices where note.lines[k].speaker == speaker { note.lines[k].speaker = other.id }+ note.speakers.remove(at: index)+ } else {+ note.speakers[index].name = n+ }+ if !isMe { learn([n]) }+ replace(note)+ }++ /// You fix a line: the words you brought in are learnt for the next calls.+ func edit(_ id: UUID, line: UUID, text: String) {+ guard var note = self.note(id), let index = note.lines.firstIndex(where: { $0.id == line }) else { return }+ let t = text.trimmingCharacters(in: .whitespacesAndNewlines)+ guard !t.isEmpty, t != note.lines[index].text else { return }+ learn(NoteBuilder.newTerms(old: note.lines[index].text, new: t))+ if note.lines[index].raw == nil { note.lines[index].raw = note.lines[index].text }+ note.lines[index].text = t+ replace(note)+ }++ /// A line said by someone else than shown (an echo left, a voice taken for another).+ func assign(_ id: UUID, line: UUID, to speaker: String) {+ guard var note = self.note(id), note.speaker(speaker) != nil,+ let index = note.lines.firstIndex(where: { $0.id == line }) else { return }+ note.lines[index].speaker = speaker+ replace(note)+ }++ func deleteLine(_ id: UUID, line: UUID) {+ guard var note = self.note(id) else { return }+ note.lines.removeAll { $0.id == line }replace(note)}+ /// Correct the transcript again, then the summary from it.+ func correctAgain(_ id: UUID) async {+ await correct(id)+ if summaryEngine != .off { await summarize(id) }+ }++ var canCorrect: Bool {+ summaryEngine == .claude && FlowCleanup.hasKey+ }++ // MARK: - Vocabulary++ private static var vocabularyURL: URL { folder.appendingPathComponent("vocabulary.json") }++ func learn(_ terms: [String]) {+ let fresh = terms.filter { term in !vocabulary.contains { $0.caseInsensitiveCompare(term) == .orderedSame } }+ guard !fresh.isEmpty else { return }+ vocabulary = Array((fresh + vocabulary).prefix(300))+ do {+ try FileManager.default.createDirectory(at: Self.folder, withIntermediateDirectories: true)+ try JSONEncoder().encode(vocabulary).write(to: Self.vocabularyURL, options: .atomic)+ } catch {+ appendAppLog("nb.log", "[Notetaker] could not save the vocabulary: \(error.localizedDescription)")+ }+ }++ private func loadVocabulary() {+ guard let data = try? Data(contentsOf: Self.vocabularyURL),+ let words = try? JSONDecoder().decode([String].self, from: data) else { return }+ vocabulary = words+ }+func toggle(_ id: UUID, todo: UUID) {guard var note = self.note(id), let index = note.todos.firstIndex(where: { $0.id == todo }) else { return }note.todos[index].done.toggle()diff --git a/docs/SPEC.md b/docs/SPEC.mdindex 70025fc4..3a84e049 100644--- a/docs/SPEC.md+++ b/docs/SPEC.md@@ -439,3 +439,15 @@ Réglages → Raccourcis (`ShortcutsSettingsView`) :- L'image arrive dans l'île (vue `.capture`) : son `gulp`, Mochi avale. Boutons : **Coller** (dans l'app où on était : ⌘V, ou ⌃V dans un terminal pour Claude Code ; le presse-papiers reprend son contenu après 1,5 s), **Copier**, **Annoter** (fenêtre : flèche, cadre, crayon, 4 couleurs, ⌘Z, l'image annotée revient dans l'île), **Demander à Claude** (chat de l'île, image jointe).- Avec Flow : tant que la capture est dans l'île (3 min max), maintenir la touche de dictée colle l'image puis les mots au relâchement.- Les captures restent dans le dossier temporaire (les 20 dernières) et ne partent vers Claude que si on les colle ou qu'on demande à Claude.++## 18. Notetaker : écho, intervenants et mots mal entendus (build GitHub)++Constaté sur un vrai appel sans casque : la voix de l'autre revenait dans le micro et passait pour la mienne, un même interlocuteur était coupé en deux personnes, et Parakeet écrivait « UG charging » pour « EV charging ».++- **Une seule horloge.** Le micro et le son du Mac comptent leur temps depuis le même instant : le son du Mac, qui démarre un peu après, commence par le silence manquant. Les lignes et l'écho s'alignent.+- **L'écho, au son.** Le son du Mac sert de référence : on mesure le délai et la force avec lesquels il revient dans le micro, puis un mot du micro qui n'a pas nettement plus d'énergie (≈ 10 dB) que cet écho est retiré. Avec un casque, l'écho est nul et rien n'est retiré ; parler par-dessus l'autre reste à moi.+- **L'écho, aux mots.** En plus : trois mots de suite que l'autre a dits dans le même ordre au même moment, et la fin de sa phrase, sont retirés de mes lignes.+- **Une voix coupée en deux.** Un « intervenant » qui ne parle que par bribes (jamais 3 s d'affilée, peu au total) est rendu à la voix la plus proche (empreinte vocale du diarizer, sinon la plus proche dans le temps).+- **Mots mal entendus.** Quand Claude écrit les résumés (clé de Réglages → Chat), il corrige d'abord la transcription : mots mal reconnus, hésitations retirées, sans reformuler ni traduire, sans inventer de nom, de numéro ni de lien, `[?]` sur ce qui reste incompréhensible. Chaque correction est vérifiée (une ligne corrigée doit rester proche de l'originale) et ce qui avait été entendu est gardé (`raw`). Il nomme aussi un intervenant quand l'appel dit qui il est ; un nom donné par l'utilisateur n'est jamais remplacé.+- **Au fil des appels.** Menu ⋯ d'une ligne (ou clic droit) : Corriger la ligne, Dit par…, Supprimer la ligne. Les mots apportés en corrigeant (noms, marques, sigles, mots techniques) et les noms donnés aux intervenants sont retenus (`Notes/vocabulary.json`, 300 max) et envoyés à Claude avec chaque transcription, avec le dictionnaire de Flow. Renommer un intervenant avec le nom d'un autre fusionne les deux.+- Menu ⋯ de la note : « Corriger à nouveau la transcription ».diff --git a/scripts/test-notetaker.sh b/scripts/test-notetaker.shindex ed61427b..b0d2020c 100755--- a/scripts/test-notetaker.sh+++ b/scripts/test-notetaker.sh@@ -3,5 +3,5 @@ set -euo pipefailcd "$(dirname "$0")/.."TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/coucou-notetaker.XXXXXX")"trap 'rm -rf "$TEST_DIR"' EXIT-swiftc NotchBuddy/Sources/App/NoteModels.swift tests/NoteModelsTests.swift -o "$TEST_DIR/notetaker-tests"+swiftc NotchBuddy/Sources/App/NoteModels.swift NotchBuddy/Sources/App/NoteEcho.swift tests/NoteModelsTests.swift -o "$TEST_DIR/notetaker-tests""$TEST_DIR/notetaker-tests"diff --git a/tests/NoteModelsTests.swift b/tests/NoteModelsTests.swiftindex 5e86eeb9..2bdbf455 100644--- a/tests/NoteModelsTests.swift+++ b/tests/NoteModelsTests.swift@@ -95,6 +95,116 @@ enum NoteModelsTests {check("clock with hours", NoteBuilder.clock(3725) == "1:02:05")}+ print("\nEcho, word by word: their sentence inside one of my lines")+ do {+ // I talk, then their sentence comes back through the speakers, heard a little differently.+ let mine = words("so what do you need", from: 0)+ + words("currently we start with one page on this requirement", from: 10)+ + words("okay perfect", from: 15)+ let theirs = words("currently we start with one page for this particular category", from: 9.9)+ let kept = NoteBuilder.removingEchoWords(mine, others: theirs)+ check("my words stay", kept.map(\.text).joined(separator: " ") == "so what do you need okay perfect")+ let lone = NoteBuilder.removingEchoWords(words("yeah sure", from: 3), others: words("yeah", from: 3))+ check("two words in common are not an echo", lone.count == 2)+ }++ print("\nEcho, by the sound: the others' voice back in the microphone")+ do {+ let rate = 16_000.0+ var seed: UInt32 = 7+ func noise() -> Float { seed = seed &* 1_664_525 &+ 1_013_904_223; return Float(seed >> 8) / Float(1 << 24) - 0.5 }+ // Syllables: 80–250 ms each, of random loudness, some silent.+ var syllables: [Float] = []+ while syllables.count < 200 { syllables.append(noise() < -0.3 ? 0.05 : 0.5 + noise()) }+ var bounds: [Double] = [0]+ while bounds.last! < 15 { bounds.append(bounds.last! + 0.165 + Double(noise()) * 0.17) }+ func voice(_ t: Double, _ f: Double, _ amp: Float) -> Float {+ let k = bounds.firstIndex { $0 > t } ?? 1+ return Float(sin(2 * .pi * f * t)) * amp * syllables[(k + Int(f)) % syllables.count]+ }+ let count = Int(rate * 14)+ var sys = [Float](repeating: 0, count: count), mic = [Float](repeating: 0, count: count)+ let delay = Int(0.08 * rate)+ for i in 0..<count {+ let t = Double(i) / rate+ // Them: 1–5 s and 9–11 s. Me: 6–7.5 s alone, and 9.3–10.2 s over them.+ if (1..<5).contains(t) || (9..<11).contains(t) { sys[i] = voice(t, 180, 0.1) }+ mic[i] = noise() * 0.002+ if (6..<7.5).contains(t) || (9.3..<10.2).contains(t) { mic[i] += voice(t, 140, 0.12) }+ }+ for i in delay..<count { mic[i] += 0.2 * sys[i - delay] }+ let heard = [TimedWord(text: "currently", start: 1.5, end: 2), TimedWord(text: "we", start: 2.1, end: 2.3),+ TimedWord(text: "start", start: 2.4, end: 2.9),+ TimedWord(text: "okay", start: 6.2, end: 6.6), TimedWord(text: "perfect", start: 6.8, end: 7.3),+ TimedWord(text: "wait", start: 9.5, end: 9.9)]+ let a = NoteEcho.analyze(mic: NoteEcho.levels(mic), sys: NoteEcho.levels(sys))+ check("the delay is found (80 ms = 4 frames)", a.map { abs($0.lag - 4) <= 1 } == true)+ check("the echo's strength is found", a.map { $0.gain > 0.1 && $0.gain < 0.4 } == true)+ let result = NoteEcho.removing(heard, micSamples: mic, sysSamples: sys)+ check("the echo is gone, my words stay, even over them",+ result.words.map(\.text) == ["okay", "perfect", "wait"])+ // With headphones: nothing comes back, nothing is removed.+ var quiet = [Float](repeating: 0, count: count)+ for i in 0..<count { quiet[i] = noise() * 0.002 + ((9.3..<10.2).contains(Double(i) / rate) ? voice(Double(i) / rate, 140, 0.12) : 0) }+ let headphones = NoteEcho.removing([TimedWord(text: "wait", start: 9.5, end: 9.9)], micSamples: quiet, sysSamples: sys)+ check("with headphones, my words stay", headphones.words.count == 1)+ }++ print("\nOne voice cut in two by the diarizer")+ do {+ let a: [Float] = [1, 0, 0.1], b: [Float] = [0, 1, 0]+ let turns = [SpeakerTurn(speaker: "S1", start: 0, end: 20, embedding: a),+ SpeakerTurn(speaker: "S2", start: 21, end: 21.8, embedding: [0.9, 0.1, 0.2]),+ SpeakerTurn(speaker: "S3", start: 23, end: 40, embedding: b),+ SpeakerTurn(speaker: "S2", start: 41, end: 41.5, embedding: [0.1, 0.8, 0])]+ let merged = NoteBuilder.mergingFragments(turns)+ check("the bits go to the closest voice", merged.map(\.speaker) == ["S1", "S1", "S3", "S3"])+ let plain = NoteBuilder.mergingFragments([SpeakerTurn(speaker: "A", start: 0, end: 30),+ SpeakerTurn(speaker: "B", start: 30.2, end: 31)])+ check("without fingerprints, the nearest in time", plain.map(\.speaker) == ["A", "A"])+ let two = NoteBuilder.mergingFragments([SpeakerTurn(speaker: "A", start: 0, end: 10),+ SpeakerTurn(speaker: "B", start: 11, end: 20)])+ check("two real voices stay two", Set(two.map(\.speaker)).count == 2)+ }++ print("\nCorrecting the transcript")+ do {+ var note = Note(title: "Call", start: Date())+ note.speakers = [NoteSpeaker(id: "me", name: "Moi", color: 0, isMe: true),+ NoteSpeaker(id: "p2", name: "Personne 2", color: 1)]+ note.lines = [NoteLine(speaker: "p2", start: 0, end: 5, text: "uh we manufacture the EU charges primarily"),+ NoteLine(speaker: "p2", start: 6, end: 9, text: "I built it with few other elemental codes right"),+ NoteLine(speaker: "me", start: 10, end: 12, text: "okay perfect Pratap"),+ NoteLine(speaker: "p2", start: 13, end: 15, text: "the price is fine")]+ check("numbered", NoteBuilder.numberedTranscript(note, range: 0..<2).hasPrefix("[0] Personne 2: uh we"))+ let reply = """+ {"lines": {"0": "We manufacture the EV chargers primarily.", "[1]": "I built it with a few Elementor shortcodes, right.",+ "3": "We will pay you one million dollars tomorrow morning."}, "speakers": {"Personne 2": "Pratap", "Moi": "Louis"}}+ """+ let correction = NoteBuilder.parseCorrection(reply)+ check("parsed", correction?.lines.count == 3 && correction?.speakers["Personne 2"] == "Pratap")+ let result = NoteBuilder.applying(correction!, to: note, personName: person)+ check("misheard words fixed", result.note.lines[0].text == "We manufacture the EV chargers primarily."+ && result.note.lines[1].text == "I built it with a few Elementor shortcodes, right.")+ check("what was heard is kept", result.note.lines[0].raw == "uh we manufacture the EU charges primarily")+ check("a rewritten line is refused", result.note.lines[3].text == "the price is fine" && result.fixed == 2)+ check("the person is named, me stays me", result.note.speakers.map(\.name) == ["Moi", "Pratap"])+ let invented = NoteBuilder.applying(.init(lines: [:], speakers: ["Personne 2": "Rahul"]), to: note, personName: person)+ check("a name never said is not given", invented.note.speakers[1].name == "Personne 2")+ var renamed = note+ renamed.speakers[1].name = "Client"+ check("your name is kept", NoteBuilder.applying(correction!, to: renamed, personName: person).note.speakers[1].name == "Client")+ check("batches", NoteBuilder.batches(note.lines, words: 10) == [0..<2, 2..<4])+ }++ print("\nWords learnt from your corrections")+ do {+ check("acronyms and brands", NoteBuilder.newTerms(old: "the UG charging with elemental codes",+ new: "the EV charging with Elementor shortcodes") == ["EV", "Elementor", "shortcodes"])+ check("small common words are not learnt", NoteBuilder.newTerms(old: "it is fine", new: "it was fine").isEmpty)+ check("a site", NoteBuilder.newTerms(old: "it's teslaws.com", new: "it's teslasvolt.com") == ["teslasvolt.com"])+ }+if failures > 0 {print("\n\(failures) failure(s)")exit(1)