Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
feat(private-ai): free model memory when idle
altic-dev523fa1c8f564
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swiftindex 12bfe81e..b1e055e9 100644--- a/Sources/Fluid/Persistence/SettingsStore.swift+++ b/Sources/Fluid/Persistence/SettingsStore.swift@@ -1692,6 +1692,15 @@ final class SettingsStore: ObservableObject {}}+ /// Experimental. Off by default: the model stays loaded as before.+ var privateAIIdleUnloadEnabled: Bool {+ get { self.defaults.bool(forKey: Keys.privateAIIdleUnloadEnabled) }+ set {+ objectWillChange.send()+ self.defaults.set(newValue, forKey: Keys.privateAIIdleUnloadEnabled)+ }+ }+var privateAIBoostEnabled: Bool {get { self.defaults.object(forKey: PrivateAIProviderFeature.shared.boostDefaultsKey) as? Bool ?? true }set {@@ -5648,6 +5657,7 @@ private extension SettingsStore {static let selectedProviderID = "SelectedProviderID"static let privateAIPrefixKVCacheEnabled = "PrivateAIProviderPrefixKVCacheEnabled"static let privateAIBoostEnabled = "PrivateAIProviderBoostEnabled"+ static let privateAIIdleUnloadEnabled = "PrivateAIProviderIdleUnloadEnabled"static let privateAIBackendPreference = SettingsStore.privateAIBackendPreferenceDefaultsKeystatic let privateAIContextTokenLimit = "PrivateAIProviderContextTokenLimit"static let privateAIContextDefaultMigratedTo4K = "PrivateAIProviderContextDefaultMigratedTo4K"diff --git a/Sources/Fluid/Services/PrivateAIIdleUnloader.swift b/Sources/Fluid/Services/PrivateAIIdleUnloader.swiftnew file mode 100644index 00000000..c64134a0--- /dev/null+++ b/Sources/Fluid/Services/PrivateAIIdleUnloader.swift@@ -0,0 +1,72 @@+import Foundation++/// Releases the local model after a quiet period so its memory goes back to+/// the system instead of being compressed or swapped. Work in flight, and any+/// state the caller reports as busy (a recording that will need the model when+/// it ends), postpones the unload rather than cancelling it.+actor PrivateAIIdleUnloader {+ private let delay: Duration+ private let isEnabled: @Sendable () async -> Bool+ private let isBusy: @Sendable () async -> Bool+ private let unload: @Sendable () async -> Void+ private var inFlight = 0+ private var pending: Task<Void, Never>?++ init(+ delay: Duration,+ isEnabled: @escaping @Sendable () async -> Bool,+ isBusy: @escaping @Sendable () async -> Bool,+ unload: @escaping @Sendable () async -> Void+ ) {+ self.delay = delay+ self.isEnabled = isEnabled+ self.isBusy = isBusy+ self.unload = unload+ }++ func begin() {+ self.inFlight += 1+ self.pending?.cancel()+ self.pending = nil+ }++ func end() {+ self.inFlight = max(0, self.inFlight - 1)+ guard self.inFlight == 0 else { return }+ self.schedule()+ }++ /// Runs `work` as model activity: the countdown restarts when it finishes.+ nonisolated func tracking<T>(_ work: () async throws -> T) async rethrows -> T {+ await self.begin()+ do {+ let result = try await work()+ await self.end()+ return result+ } catch {+ await self.end()+ throw error+ }+ }++ private func schedule() {+ self.pending?.cancel()+ self.pending = Task { [delay] in+ try? await Task.sleep(for: delay)+ guard !Task.isCancelled else { return }+ await self.fire()+ }+ }++ private func fire() async {+ guard self.inFlight == 0, await self.isEnabled() else { return }+ if await self.isBusy() {+ self.schedule()+ return+ }+ // A request may have started while the checks above were suspended.+ guard self.inFlight == 0, !Task.isCancelled else { return }+ self.pending = nil+ await self.unload()+ }+}diff --git a/Sources/Fluid/Services/PrivateAIIntegrationService.swift b/Sources/Fluid/Services/PrivateAIIntegrationService.swiftindex 2b9dc2c4..24cf3629 100644--- a/Sources/Fluid/Services/PrivateAIIntegrationService.swift+++ b/Sources/Fluid/Services/PrivateAIIntegrationService.swift@@ -237,8 +237,17 @@ actor PrivateAIIntegrationService {await Self.provider.loadedModelState()}+ /// Experimental: gives the model's memory back after a quiet period. The+ /// next dictation reloads it while the user is still speaking.+ nonisolated static let idleUnloader = PrivateAIIdleUnloader(+ delay: .seconds(600),+ isEnabled: { await MainActor.run { SettingsStore.shared.privateAIIdleUnloadEnabled } },+ isBusy: { await MainActor.run { AppServices.shared.asr.isRunningOrStarting } },+ unload: { await PrivateAIIntegrationService.shared.unloadCachedRuntime(reason: "idle") }+ )+func loadModel(_ model: PrivateAIRegisteredModel) async throws -> PrivateAIStatus {- let status = try await Self.provider.loadModel(model)+ let status = try await Self.idleUnloader.tracking { try await Self.provider.loadModel(model) }guard status.state == .ready else { return status }await self.removeInactiveInstalledModels(keeping: model)@@ -246,7 +255,7 @@ actor PrivateAIIntegrationService {}func verifyModel(_ model: PrivateAIRegisteredModel) async throws -> PrivateAIStatus {- try await Self.provider.verifyModel(model)+ try await Self.idleUnloader.tracking { try await Self.provider.verifyModel(model) }}func removeInactiveInstalledModels(keeping model: PrivateAIRegisteredModel) async {@@ -272,7 +281,7 @@ actor PrivateAIIntegrationService {}func prewarmDictation() async {- await Self.provider.prewarmDictation()+ await Self.idleUnloader.tracking { await Self.provider.prewarmDictation() }}func unloadCachedRuntime(reason: String = "manual") async {@@ -289,12 +298,14 @@ actor PrivateAIIntegrationService {context: AppContext) async throws -> EnhancementResult {let budget = try Self.validatedDictationBudget(inputText, contextTokenLimit: runtime.contextTokenLimit)- return try await self.dictationProvider.enhanceDictation(- inputText,- runtime: runtime,- context: context,- maxOutputTokens: budget.maxOutputTokens- )+ return try await Self.idleUnloader.tracking {+ try await self.dictationProvider.enhanceDictation(+ inputText,+ runtime: runtime,+ context: context,+ maxOutputTokens: budget.maxOutputTokens+ )+ }}nonisolated func enhanceDictation(@@ -304,13 +315,15 @@ actor PrivateAIIntegrationService {streamHandler: PrivateAIStreamHandler?) async throws -> EnhancementResult {let budget = try Self.validatedDictationBudget(inputText, contextTokenLimit: runtime.contextTokenLimit)- return try await self.dictationProvider.enhanceDictation(- inputText,- runtime: runtime,- context: context,- maxOutputTokens: budget.maxOutputTokens,- streamHandler: streamHandler- )+ return try await Self.idleUnloader.tracking {+ try await self.dictationProvider.enhanceDictation(+ inputText,+ runtime: runtime,+ context: context,+ maxOutputTokens: budget.maxOutputTokens,+ streamHandler: streamHandler+ )+ }}private nonisolated static func validatedDictationBudget(@@ -333,12 +346,14 @@ actor PrivateAIIntegrationService {runtime: RuntimeConfiguration,context: AppContext) async throws -> EnhancementResult {- try await Self.provider.rewrite(- inputText,- systemPrompt: systemPrompt,- runtime: runtime,- context: context- )+ try await Self.idleUnloader.tracking {+ try await Self.provider.rewrite(+ inputText,+ systemPrompt: systemPrompt,+ runtime: runtime,+ context: context+ )+ }}}diff --git a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swiftindex 271d5e27..e178cf49 100644--- a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift+++ b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift@@ -999,6 +999,13 @@ extension AIEnhancementSettingsView {)}+ private var privateAIIdleUnloadBinding: Binding<Bool> {+ Binding(+ get: { self.settings.privateAIIdleUnloadEnabled },+ set: { self.settings.privateAIIdleUnloadEnabled = $0 }+ )+ }+private var privateAIBoostBinding: Binding<Bool> {Binding(get: { self.settings.privateAIBoostEnabled },@@ -1914,6 +1921,11 @@ extension AIEnhancementSettingsView {Toggle("Faster first result", isOn: self.privateAIPrefixCacheBinding).labelsHidden().toggleStyle(.switch).disabled(isBusy)}+ Divider()+ FluidManagementRow(title: "Free memory when idle", detail: "Experimental. Unloads the model after 10 quiet minutes and reloads it as you start speaking.") {+ Toggle("Free memory when idle", isOn: self.privateAIIdleUnloadBinding)+ .labelsHidden().toggleStyle(.switch)+ }if self.privateAIShowsBoostRow {Divider()FluidManagementRow(title: "Faster results", detail: "Extra local acceleration. Uses more memory.") {diff --git a/Tests/PrivateAIIdleUnloaderTests.swift b/Tests/PrivateAIIdleUnloaderTests.swiftnew file mode 100644index 00000000..4b579c25--- /dev/null+++ b/Tests/PrivateAIIdleUnloaderTests.swift@@ -0,0 +1,112 @@+import Foundation++private actor Recorder {+ private(set) var unloads = 0+ var enabled = true+ var busy = false+ func noteUnload() { self.unloads += 1 }+ func set(enabled: Bool) { self.enabled = enabled }+ func set(busy: Bool) { self.busy = busy }+}++private func makeUnloader(_ recorder: Recorder, delayMs: Int = 60) -> PrivateAIIdleUnloader {+ PrivateAIIdleUnloader(+ delay: .milliseconds(delayMs),+ isEnabled: { await recorder.enabled },+ isBusy: { await recorder.busy },+ unload: { await recorder.noteUnload() }+ )+}++private func pause(_ ms: Int) async {+ try? await Task.sleep(for: .milliseconds(ms))+}++private func expect(_ condition: Bool, _ message: String) {+ guard condition else {+ FileHandle.standardError.write(Data("FAIL: \(message)\n".utf8))+ exit(1)+ }+ print("ok: \(message)")+}++@main+enum PrivateAIIdleUnloaderTests {+ static func main() async {+ do {+ let recorder = Recorder()+ let unloader = makeUnloader(recorder)+ await unloader.tracking {}+ await pause(250)+ let unloads = await recorder.unloads+ expect(unloads == 1, "unloads once after the quiet period")+ }+ do {+ let recorder = Recorder()+ let unloader = makeUnloader(recorder, delayMs: 150)+ for _ in 0 ..< 4 {+ await unloader.tracking {}+ await pause(60)+ }+ var unloads = await recorder.unloads+ expect(unloads == 0, "activity keeps restarting the countdown")+ await pause(400)+ unloads = await recorder.unloads+ expect(unloads == 1, "unloads after activity stops")+ }+ do {+ let recorder = Recorder()+ let unloader = makeUnloader(recorder)+ await unloader.begin()+ await pause(250)+ var unloads = await recorder.unloads+ expect(unloads == 0, "never unloads while a request is in flight")+ await unloader.end()+ await pause(250)+ unloads = await recorder.unloads+ expect(unloads == 1, "unloads once the request finishes")+ }+ do {+ let recorder = Recorder()+ let unloader = makeUnloader(recorder)+ await unloader.begin()+ await unloader.tracking {}+ await pause(250)+ let unloads = await recorder.unloads+ expect(unloads == 0, "overlapping requests hold the model until the last ends")+ await unloader.end()+ }+ do {+ let recorder = Recorder()+ await recorder.set(enabled: false)+ let unloader = makeUnloader(recorder)+ await unloader.tracking {}+ await pause(250)+ let unloads = await recorder.unloads+ expect(unloads == 0, "does nothing when the setting is off")+ }+ do {+ let recorder = Recorder()+ await recorder.set(busy: true)+ let unloader = makeUnloader(recorder)+ await unloader.tracking {}+ await pause(250)+ var unloads = await recorder.unloads+ expect(unloads == 0, "waits while a recording is in progress")+ await recorder.set(busy: false)+ await pause(250)+ unloads = await recorder.unloads+ expect(unloads == 1, "unloads after the recording ends")+ }+ do {+ struct Boom: Error {}+ let recorder = Recorder()+ let unloader = makeUnloader(recorder)+ do { try await unloader.tracking { throw Boom() } } catch {}+ await pause(250)+ let unloads = await recorder.unloads+ expect(unloads == 1, "a failed request still releases its hold")+ }+ print("PrivateAIIdleUnloaderTests passed")+ }+}diff --git a/Tests/run_private_ai_idle_unloader_tests.sh b/Tests/run_private_ai_idle_unloader_tests.shnew file mode 100755index 00000000..30e3aa7b--- /dev/null+++ b/Tests/run_private_ai_idle_unloader_tests.sh@@ -0,0 +1,16 @@+#!/bin/sh+set -eu++task_developer_dir="${DEVELOPER_DIR:-$(xcode-select -p)}"+case "$task_developer_dir" in+ */Xcode*.app/Contents/Developer) ;;+ *) echo "Set DEVELOPER_DIR to an installed full Xcode before running tests." >&2; exit 1 ;;+esac+export DEVELOPER_DIR="$task_developer_dir"+task_repo_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)+task_test_dir=$(mktemp -d /tmp/fluidvoice-idle-unloader-tests.XXXXXX)+xcrun swiftc -O -parse-as-library \+ "$task_repo_dir/Sources/Fluid/Services/PrivateAIIdleUnloader.swift" \+ "$task_repo_dir/Tests/PrivateAIIdleUnloaderTests.swift" \+ -o "$task_test_dir/idle-unloader-tests"+"$task_test_dir/idle-unloader-tests"