Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
Let Edit Mode read the front window
altic-devdeb28f1dd2bb
diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swiftindex 2e921d38..dcf45e5b 100644--- a/Sources/Fluid/ContentView.swift+++ b/Sources/Fluid/ContentView.swift@@ -4672,6 +4672,7 @@ struct ContentView: View {self.rewriteModeService.startWithoutSelection()}}+ self.startReadingEditWindow()self.menuBarManager.setOverlayMode(.edit)case .write, .rewrite:guard self.activeRecordingMode != .edit || NotchContentState.shared.mode == .dictation else { return }@@ -4685,10 +4686,24 @@ struct ContentView: View {self.rewriteModeService.startWithoutSelection()}}+ self.startReadingEditWindow()self.menuBarManager.setOverlayMode(.edit)}}+ /// Reads the front window for on-device Edit Mode. An API never gets it.+ private func startReadingEditWindow() {+ guard self.settings.editModeReadsScreen,+ self.settings.effectiveRewriteModeProviderID == PrivateAIProviderFeature.shared.providerID,+ let target = NotchContentState.shared.recordingTargetContext,+ target.pid != ProcessInfo.processInfo.processIdentifier+ else {+ self.rewriteModeService.stopReadingWindow()+ return+ }+ self.rewriteModeService.startReadingWindow(target: target)+ }+private func handleLiveOverlayModeSwitch(_ mode: OverlayMode) {guard !NotchContentState.shared.isProcessing else { return }switch mode {@@ -5065,6 +5080,7 @@ struct ContentView: View {// Text was selected - edit mode (with selected context)self.menuBarManager.setOverlayMode(.edit)}+ self.startReadingEditWindow()// Set flag so stopAndProcessTranscription knows to process as rewriteself.setActiveRecordingMode(.edit)diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swiftindex 620673eb..2ac732b4 100644--- a/Sources/Fluid/Persistence/SettingsStore.swift+++ b/Sources/Fluid/Persistence/SettingsStore.swift@@ -3457,6 +3457,15 @@ final class SettingsStore: ObservableObject {}}+ /// Lets on-device Edit Mode use the text in the front window. On until turned off.+ var editModeReadsScreen: Bool {+ get { self.defaults.object(forKey: Keys.editModeReadsScreen) as? Bool ?? true }+ set {+ objectWillChange.send()+ self.defaults.set(newValue, forKey: Keys.editModeReadsScreen)+ }+ }+/// Global check if a model is a reasoning model (requires special params/max_completion_tokens)func isReasoningModel(_ model: String) -> Bool {// Drop a provider/namespace prefix (e.g. OpenRouter's "openai/") so the@@ -6072,6 +6081,7 @@ private extension SettingsStore {// Model Reasoning Config Keysstatic let modelReasoningConfigs = "ModelReasoningConfigs"static let rewriteModeShortcutEnabled = "RewriteModeShortcutEnabled"+ static let editModeReadsScreen = "EditModeReadsScreen"static let showThinkingTokens = "ShowThinkingTokens"// Stats Keysdiff --git a/Sources/Fluid/Services/DictationContextCapture.swift b/Sources/Fluid/Services/DictationContextCapture.swiftindex 56f36f5d..6dc5fa22 100644--- a/Sources/Fluid/Services/DictationContextCapture.swift+++ b/Sources/Fluid/Services/DictationContextCapture.swift@@ -12,6 +12,19 @@ nonisolated struct DictationTypingContext: Sendable, Equatable {// One Accessibility element or OCR line per entry; nil means none was read.// swiftlint:disable:next discouraged_optional_collectionvar screenText: [String]?+ // The window as Edit Mode reads it; nil for a dictation.+ // swiftlint:disable:next discouraged_optional_collection+ var windowLines: [EditModeWindowLine]? = nil+}++/// A piece of text in the front window and where it is. Stays on this Mac and is+/// never logged or kept.+nonisolated struct EditModeWindowLine: Sendable, Equatable {+ let text: String+ let x: Int+ let y: Int+ let width: Int+ let height: Int}/// Reads the dictation context when recording starts, off the main thread,@@ -63,6 +76,24 @@ final nonisolated class DictationContextCapture: @unchecked Sendable {return DictationContextCapture(target: target, task: task)}+ /// Reads the window for on-device Edit Mode, which has its own setting.+ @MainActor+ static func startForEditMode(target: TypingService.RecordingTargetContext) -> DictationContextCapture? {+ guard AXIsProcessTrusted() else { return nil }+ let request = DictationContextReader.Request(+ pid: target.pid,+ bundleID: target.bundleIdentifier,+ window: target.window,+ element: target.element,+ textBeforeCursor: nil,+ isEnabled: true+ )+ let task = Task.detached(priority: .userInitiated) {+ await DictationContextReader.readWindowLines(request)+ }+ return DictationContextCapture(target: target, task: task)+ }+/// The context, or nil when it is not ready within `timeout`; a late/// result is dropped.func result(timeout: Duration) async -> DictationTypingContext? {@@ -161,6 +192,11 @@ nonisolated enum DictationContextReader {private static let messagingTimeoutSeconds: Float = 0.1/// Omit long text values when the app cannot report their visible range.private static let longValueCharacters = 4000+ /// Edit Mode isn't waiting on the read while the user speaks, so it can look further.+ private static let windowBudgetSeconds = 1.5+ private static let maxVisitedWindowElements = 10_000+ private static let windowReadAttempts = 3+ private static let windowReadRetryDelay: Duration = .milliseconds(300)private static let passwordManagerMarkers = ["password", "1password", "onepassword", "bitwarden", "lastpass", "dashlane",@@ -209,6 +245,35 @@ nonisolated enum DictationContextReader {return context.textBeforeCursor == nil && context.screenText == nil ? nil : context}+ /// The whole window for Edit Mode, with positions. A dictation only wants rare+ /// words; this keeps numbers, repeats and long paragraphs, because the answer+ /// comes from them.+ static func readWindowLines(+ _ request: Request,+ recognizeWindow: @Sendable (pid_t, AXUIElement?) async -> [EditModeWindowLine] = { await DictationContextReader.recognizedWindowLines(pid: $0, window: $1) }+ ) async -> DictationTypingContext? {+ guard request.isEnabled, !Task.isCancelled else { return nil }+ if let element = request.element { AXUIElementSetMessagingTimeout(element, self.messagingTimeoutSeconds) }+ guard !self.isPrivate(request), let window = self.windows(for: request).first else { return nil }++ // Web content builds its Accessibility tree only once asked, so the first+ // press in such an app may need another look.+ var lines = EditModeWindowLines()+ for attempt in 0..<self.windowReadAttempts where lines.wordCount < self.minimumAccessibilityWords {+ if attempt > 0 { try? await Task.sleep(for: self.windowReadRetryDelay) }+ guard !Task.isCancelled else { return nil }+ lines = EditModeWindowLines()+ self.collectWindowLines(in: window, into: &lines)+ }+ var read = lines.lines+ if lines.wordCount < self.minimumAccessibilityWords {+ let recognized = await recognizeWindow(request.pid, window)+ if !recognized.isEmpty { read = recognized }+ }+ guard !Task.isCancelled, !read.isEmpty else { return nil }+ return DictationTypingContext(textBeforeCursor: nil, screenText: nil, windowLines: read)+ }+private static func windows(for request: Request) -> [AXUIElement] {guard let window = request.window else { return [] }let app = AXUIElementCreateApplication(request.pid)@@ -314,6 +379,34 @@ nonisolated enum DictationContextReader {}}+ private static func collectWindowLines(in window: AXUIElement, into lines: inout EditModeWindowLines) {+ let deadline = ProcessInfo.processInfo.systemUptime + self.windowBudgetSeconds+ AXUIElementSetMessagingTimeout(window, self.messagingTimeoutSeconds)+ let windowFrame = self.frame(of: window)+ var stack = [window]+ var visited = 0+ while let element = stack.popLast(), visited < self.maxVisitedWindowElements, !lines.isFull {+ visited += 1+ if Task.isCancelled || ProcessInfo.processInfo.systemUptime > deadline {+ break+ }+ AXUIElementSetMessagingTimeout(element, self.messagingTimeoutSeconds)+ guard let item = self.elementText(element),+ !item.isSecure, !self.skippedRoles.contains(item.role)+ else { continue }+ if let windowFrame, let frame = item.frame, !frame.isEmpty, !frame.intersects(windowFrame) {+ continue+ }+ // Without a position the text can't be placed, and it is rarely on screen.+ if let frame = item.frame, !frame.isEmpty {+ for text in item.texts {+ lines.add(text, frame: frame)+ }+ }+ stack.append(contentsOf: item.children.prefix(max(0, self.maxVisitedWindowElements - visited - stack.count)).reversed())+ }+ }+private static func elementText(_ element: AXUIElement) -> ElementText? {var valuesRef: CFArray?guard AXUIElementCopyMultipleAttributeValues(element, self.fetchedAttributes, AXCopyMultipleAttributeOptions(), &valuesRef) == .success,@@ -365,15 +458,37 @@ nonisolated enum DictationContextReader {/// (Vision garbles small gray text at 1x), and reads it with Vision./// Needs Screen Recording; FluidVoice never asks for it here.private static func recognizedWindowText(pid: pid_t, window: AXUIElement?) async -> [String] {+ guard let image = await self.windowImage(pid: pid, window: window) else { return [] }+ return self.recognizedText(in: image).compactMap { $0.topCandidates(1).first?.string }+ }++ /// The same read as `recognizedWindowText`, keeping where each line is.+ private static func recognizedWindowLines(pid: pid_t, window: AXUIElement?) async -> [EditModeWindowLine] {+ guard let image = await self.windowImage(pid: pid, window: window) else { return [] }+ return self.recognizedText(in: image).compactMap { observation in+ guard let text = observation.topCandidates(1).first?.string else { return nil }+ let box = VNImageRectForNormalizedRect(observation.boundingBox, image.width, image.height)+ // Vision counts from the bottom of the image, screen text from the top.+ return EditModeWindowLine(+ text: text,+ x: Int(box.minX),+ y: image.height - Int(box.maxY),+ width: Int(box.width),+ height: Int(box.height)+ )+ }+ }++ private static func windowImage(pid: pid_t, window: AXUIElement?) async -> CGImage? {guard !Task.isCancelled, CGPreflightScreenCaptureAccess(),let frame = window.flatMap(self.frame(of:)),let windowID = self.windowID(pid: pid, frame: frame)- else { return [] }+ else { return nil }do {let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)guard !Task.isCancelled,let scWindow = content.windows.first(where: { $0.windowID == windowID })- else { return [] }+ else { return nil }let filter = SCContentFilter(desktopIndependentWindow: scWindow)let configuration = SCStreamConfiguration()let scale = CGFloat(filter.pointPixelScale)@@ -383,14 +498,13 @@ nonisolated enum DictationContextReader {configuration.showsCursor = falseconfiguration.ignoreShadowsSingleWindow = truelet image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)- guard !Task.isCancelled else { return [] }- return self.recognizedLines(in: image)+ return Task.isCancelled ? nil : image} catch {- return []+ return nil}}- private static func recognizedLines(in image: CGImage) -> [String] {+ private static func recognizedText(in image: CGImage) -> [VNRecognizedTextObservation] {let request = VNRecognizeTextRequest()request.recognitionLevel = .accuraterequest.usesLanguageCorrection = false@@ -400,7 +514,7 @@ nonisolated enum DictationContextReader {} catch {return []}- return (request.results ?? []).compactMap { $0.topCandidates(1).first?.string }+ return request.results ?? []}/// Only the captured window may be read; a stale window must never fall back to another one.@@ -486,6 +600,46 @@ nonisolated enum DictationContextReader {}}+/// Window text with positions, within size caps well above what the model can+/// read: Fluid Intelligence lays it out and shortens it to fit. A link and the text+/// inside it report the same words, so a repeat only counts where it overlaps.+nonisolated struct EditModeWindowLines {+ static let maxLines = 1500+ static let maxCharacters = 60_000+ static let maxLineCharacters = 4000+ /// A container's title sits at its top, not in the middle of its contents.+ static let maxRowHeight: CGFloat = 40++ private(set) var lines: [EditModeWindowLine] = []+ private(set) var wordCount = 0+ private var characterCount = 0+ private var placed: [String: [CGRect]] = [:]++ var isFull: Bool { self.lines.count >= Self.maxLines || self.characterCount >= Self.maxCharacters }++ mutating func add(_ text: String, frame: CGRect) {+ let rows = text.split(whereSeparator: \.isNewline)+ .map { String($0.trimmingCharacters(in: .whitespaces).prefix(Self.maxLineCharacters)) }+ .filter { $0.contains { $0.isLetter || $0.isNumber } }+ guard !rows.isEmpty else { return }+ let rowHeight = min(frame.height / CGFloat(rows.count), Self.maxRowHeight)+ for (index, row) in rows.enumerated() where !self.isFull {+ let rect = CGRect(x: frame.minX, y: frame.minY + CGFloat(index) * rowHeight, width: frame.width, height: rowHeight)+ guard !(self.placed[row]?.contains { $0.intersects(rect) } ?? false) else { continue }+ self.placed[row, default: []].append(rect)+ self.lines.append(EditModeWindowLine(+ text: row,+ x: Int(rect.minX),+ y: Int(rect.minY),+ width: Int(rect.width),+ height: max(Int(rect.height), 1)+ ))+ self.wordCount += row.split(whereSeparator: \.isWhitespace).count+ self.characterCount += row.count+ }+ }+}+/// Screen lines in reading order, without repeats, within the size caps.private nonisolated struct LineCollector {private(set) var lines: [String] = []diff --git a/Sources/Fluid/Services/PrivateAIIntegrationService.swift b/Sources/Fluid/Services/PrivateAIIntegrationService.swiftindex e34a5241..efcfd335 100644--- a/Sources/Fluid/Services/PrivateAIIntegrationService.swift+++ b/Sources/Fluid/Services/PrivateAIIntegrationService.swift@@ -36,6 +36,11 @@ actor PrivateAIIntegrationService {var screenText: [String]? = nil/// The text an Edit Mode instruction applies to, when any was selected.var selectedText: String? = nil+ /// The front window, for Edit Mode models that read it.+ // swiftlint:disable:next discouraged_optional_collection+ var windowLines: [EditModeWindowLine]? = nil+ /// Signs what Edit Mode writes.+ var userDisplayName: String? = nil}struct EnhancementResult: Sendable, Equatable {diff --git a/Sources/Fluid/Services/RewriteModeService.swift b/Sources/Fluid/Services/RewriteModeService.swiftindex 8ff3f798..f6797eda 100644--- a/Sources/Fluid/Services/RewriteModeService.swift+++ b/Sources/Fluid/Services/RewriteModeService.swift@@ -15,6 +15,8 @@ final class RewriteModeService: ObservableObject {@Published private(set) var lastErrorMessage: String?@Published var isWriteMode: Bool = false // true = no text selected (write/improve), false = text selected (rewrite)private var promptAppBundleID: String?+ /// The front window as read when the shortcut was pressed.+ private var windowCapture: DictationContextCapture?private let textSelectionService = TextSelectionService.sharedprivate let typingService = TypingService()@@ -191,6 +193,19 @@ final class RewriteModeService: ObservableObject {self.thinkingBuffer = []self.promptAppBundleID = nilself.lastErrorMessage = nil+ self.stopReadingWindow()+ }++ /// Starts reading the window while it is still in front, so the answer can use+ /// what the user is looking at. Replaces the reading of any earlier press.+ func startReadingWindow(target: TypingService.RecordingTargetContext) {+ self.windowCapture?.cancel()+ self.windowCapture = DictationContextCapture.startForEditMode(target: target)+ }++ func stopReadingWindow() {+ self.windowCapture?.cancel()+ self.windowCapture = nil}func setPromptAppBundleID(_ bundleID: String?) {@@ -303,6 +318,12 @@ final class RewriteModeService: ObservableObject {if usesPrivateAIProvider || PrivateAIIntegrationService.shouldHandleDictation(model: model) {// A follow-up applies to the last result, not to the original selection.let textToEdit = self.rewrittenText.isEmpty ? self.selectedContextText : self.rewrittenText+ let window = await self.windowCapture?.result(timeout: .seconds(2))+ // Counts only: what was read is never logged.+ DebugLogger.shared.info(+ "Edit Mode read the window | lines=\(window?.windowLines?.count ?? 0) | hasSelection=\(!textToEdit.isEmpty)",+ source: "RewriteModeService"+ )let response = try await PrivateAIIntegrationService.shared.rewrite(instruction,systemPrompt: systemPrompt,@@ -322,7 +343,9 @@ final class RewriteModeService: ObservableObject {bundleID: appBundleID ?? "",windowTitle: "",appVersion: Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String,- selectedText: textToEdit.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : textToEdit+ selectedText: textToEdit.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : textToEdit,+ windowLines: window?.windowLines,+ userDisplayName: NSFullUserName()))return response.outputTextdiff --git a/Sources/Fluid/UI/AISettingsView+AdvancedSettings.swift b/Sources/Fluid/UI/AISettingsView+AdvancedSettings.swiftindex e13cc872..cd510ccb 100644--- a/Sources/Fluid/UI/AISettingsView+AdvancedSettings.swift+++ b/Sources/Fluid/UI/AISettingsView+AdvancedSettings.swift@@ -1346,7 +1346,7 @@ extension AIEnhancementSettingsView {let providers = self.editModeVerifiedProvidersreturn self.stylesSection(title: "Edit Mode",- subtitle: "Select text, press the shortcut, and say what to change. It runs on its own model, separate from dictation."+ subtitle: "Press the shortcut and say what you want. Select text first to change it. It runs on its own model, separate from dictation.") {ThemedCard(style: .subtle, padding: 16) {VStack(alignment: .leading, spacing: 14) {@@ -1368,6 +1368,11 @@ extension AIEnhancementSettingsView {Divider()if self.isPrivateAIProviderID(providerID) {EditModeLocalModelRow()+ Divider()+ FluidManagementRow(title: "Read my screen", detail: "Lets it use the text in your front window. It stays on this Mac.") {+ Toggle("Read my screen", isOn: self.$settings.editModeReadsScreen)+ .labelsHidden().toggleStyle(.switch)+ }} else {self.editModeProviderModelRow(providerID: providerID)}@@ -1375,7 +1380,7 @@ extension AIEnhancementSettingsView {FluidManagementRow(title: "Shortcut",detail: self.settings.rewriteModeShortcutEnabled- ? "Press it with text selected."+ ? "Press it and say what you want.": "Off. Turn it on in Settings to use Edit Mode.") {HStack(spacing: 10) {diff --git a/Tests/FluidDictationIntegrationTests/DictationContextCaptureTests.swift b/Tests/FluidDictationIntegrationTests/DictationContextCaptureTests.swiftindex 4392e820..1e1d5bcc 100644--- a/Tests/FluidDictationIntegrationTests/DictationContextCaptureTests.swift+++ b/Tests/FluidDictationIntegrationTests/DictationContextCaptureTests.swift@@ -62,6 +62,87 @@ final class DictationContextCaptureTests: XCTestCase {XCTAssertNil(context)}+ func testEditModeWindowReadUsesOCRPositionsWhenAppTextIsMissing() async {+ let request = DictationContextReader.Request(+ pid: 999_999,+ bundleID: "test.context",+ window: AXUIElementCreateApplication(999_999),+ element: nil,+ textBeforeCursor: nil,+ isEnabled: true+ )+ let line = EditModeWindowLine(text: "Friday at 7:30", x: 10, y: 20, width: 100, height: 14)+ let context = await DictationContextReader.readWindowLines(request) { _, _ in [line] }+ XCTAssertEqual(context?.windowLines, [line])+ XCTAssertNil(context?.screenText)+ }++ func testEditModeWindowReadSkipsPasswordManagersAndMissingWindows() async {+ let privateApp = await DictationContextReader.readWindowLines(self.request(bundleID: "com.1password", isEnabled: true)) { _, _ in+ XCTFail("Private apps must not invoke OCR")+ return []+ }+ XCTAssertNil(privateApp)+ let noWindow = await DictationContextReader.readWindowLines(self.request(isEnabled: true)) { _, _ in+ XCTFail("A missing target window must not fall back to another window")+ return []+ }+ XCTAssertNil(noWindow)+ }++ func testEditModeWindowLinesKeepNumbersAndRepeatsButNotOverlappingCopies() {+ var lines = EditModeWindowLines()+ let frame = CGRect(x: 10, y: 20, width: 200, height: 20)+ lines.add("$42.10", frame: frame)+ lines.add("$42.10", frame: frame.insetBy(dx: 2, dy: 2)) // A link and the text inside it.+ lines.add("$42.10", frame: frame.offsetBy(dx: 0, dy: 100)) // Another row of a table.+ lines.add("•", frame: frame.offsetBy(dx: 0, dy: 200))+ XCTAssertEqual(lines.lines.map(\.text), ["$42.10", "$42.10"])+ XCTAssertEqual(lines.lines.map(\.y), [20, 120])+ }++ func testEditModeWindowLinesSplitAnElementIntoRowsAndKeepTitlesAtTheTop() {+ var lines = EditModeWindowLines()+ lines.add("one\ntwo", frame: CGRect(x: 0, y: 100, width: 300, height: 40))+ XCTAssertEqual(lines.lines.map(\.y), [100, 120])+ XCTAssertEqual(lines.lines.map(\.height), [20, 20])++ var window = EditModeWindowLines()+ window.add("Inbox - Mail", frame: CGRect(x: 0, y: 0, width: 1000, height: 800))+ XCTAssertEqual(window.lines.map(\.y), [0])+ XCTAssertEqual(window.lines.map(\.height), [40])+ }++ func testEditModeWindowLinesStopAtTheirCaps() {+ var lines = EditModeWindowLines()+ for row in 0..<(EditModeWindowLines.maxLines + 10) {+ lines.add("row \(row)", frame: CGRect(x: 0, y: row * 20, width: 100, height: 20))+ }+ XCTAssertEqual(lines.lines.count, EditModeWindowLines.maxLines)++ var long = EditModeWindowLines()+ let text = String(repeating: "word ", count: EditModeWindowLines.maxLineCharacters)+ for row in 0..<100 {+ long.add("\(row) \(text)", frame: CGRect(x: 0, y: row * 20, width: 100, height: 20))+ }+ XCTAssertTrue(long.isFull)+ XCTAssertEqual(long.lines.first?.text.count, EditModeWindowLines.maxLineCharacters)+ XCTAssertLessThan(long.lines.count, 100)+ }++ func testEditModeReadsTheScreenUntilTurnedOff() {+ let key = "EditModeReadsScreen"+ let defaults = UserDefaults.standard+ let saved = defaults.object(forKey: key)+ defer {+ if let saved { defaults.set(saved, forKey: key) } else { defaults.removeObject(forKey: key) }+ }+ defaults.removeObject(forKey: key)+ XCTAssertTrue(SettingsStore.shared.editModeReadsScreen)+ SettingsStore.shared.editModeReadsScreen = false+ XCTAssertFalse(SettingsStore.shared.editModeReadsScreen)+ }+func testCursorContextIsBounded() async {let request = DictationContextReader.Request(pid: 999_999,