Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat(private-ai): free model memory when idle

altic-dev523fa1c8f564
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index 12bfe81e..b1e055e9 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -1692,6 +1692,15 @@ final class SettingsStore: ObservableObject {
}
}
+ /// Experimental. Off by default: the model stays loaded as before.
+ var privateAIIdleUnloadEnabled: Bool {
+ get { self.defaults.bool(forKey: Keys.privateAIIdleUnloadEnabled) }
+ set {
+ objectWillChange.send()
+ self.defaults.set(newValue, forKey: Keys.privateAIIdleUnloadEnabled)
+ }
+ }
+
var privateAIBoostEnabled: Bool {
get { self.defaults.object(forKey: PrivateAIProviderFeature.shared.boostDefaultsKey) as? Bool ?? true }
set {
@@ -5648,6 +5657,7 @@ private extension SettingsStore {
static let selectedProviderID = "SelectedProviderID"
static let privateAIPrefixKVCacheEnabled = "PrivateAIProviderPrefixKVCacheEnabled"
static let privateAIBoostEnabled = "PrivateAIProviderBoostEnabled"
+ static let privateAIIdleUnloadEnabled = "PrivateAIProviderIdleUnloadEnabled"
static let privateAIBackendPreference = SettingsStore.privateAIBackendPreferenceDefaultsKey
static let privateAIContextTokenLimit = "PrivateAIProviderContextTokenLimit"
static let privateAIContextDefaultMigratedTo4K = "PrivateAIProviderContextDefaultMigratedTo4K"
diff --git a/Sources/Fluid/Services/PrivateAIIdleUnloader.swift b/Sources/Fluid/Services/PrivateAIIdleUnloader.swift
new file mode 100644
index 00000000..c64134a0
--- /dev/null
+++ b/Sources/Fluid/Services/PrivateAIIdleUnloader.swift
@@ -0,0 +1,72 @@
+import Foundation
+
+/// Releases the local model after a quiet period so its memory goes back to
+/// the system instead of being compressed or swapped. Work in flight, and any
+/// state the caller reports as busy (a recording that will need the model when
+/// it ends), postpones the unload rather than cancelling it.
+actor PrivateAIIdleUnloader {
+ private let delay: Duration
+ private let isEnabled: @Sendable () async -> Bool
+ private let isBusy: @Sendable () async -> Bool
+ private let unload: @Sendable () async -> Void
+ private var inFlight = 0
+ private var pending: Task<Void, Never>?
+
+ init(
+ delay: Duration,
+ isEnabled: @escaping @Sendable () async -> Bool,
+ isBusy: @escaping @Sendable () async -> Bool,
+ unload: @escaping @Sendable () async -> Void
+ ) {
+ self.delay = delay
+ self.isEnabled = isEnabled
+ self.isBusy = isBusy
+ self.unload = unload
+ }
+
+ func begin() {
+ self.inFlight += 1
+ self.pending?.cancel()
+ self.pending = nil
+ }
+
+ func end() {
+ self.inFlight = max(0, self.inFlight - 1)
+ guard self.inFlight == 0 else { return }
+ self.schedule()
+ }
+
+ /// Runs `work` as model activity: the countdown restarts when it finishes.
+ nonisolated func tracking<T>(_ work: () async throws -> T) async rethrows -> T {
+ await self.begin()
+ do {
+ let result = try await work()
+ await self.end()
+ return result
+ } catch {
+ await self.end()
+ throw error
+ }
+ }
+
+ private func schedule() {
+ self.pending?.cancel()
+ self.pending = Task { [delay] in
+ try? await Task.sleep(for: delay)
+ guard !Task.isCancelled else { return }
+ await self.fire()
+ }
+ }
+
+ private func fire() async {
+ guard self.inFlight == 0, await self.isEnabled() else { return }
+ if await self.isBusy() {
+ self.schedule()
+ return
+ }
+ // A request may have started while the checks above were suspended.
+ guard self.inFlight == 0, !Task.isCancelled else { return }
+ self.pending = nil
+ await self.unload()
+ }
+}
diff --git a/Sources/Fluid/Services/PrivateAIIntegrationService.swift b/Sources/Fluid/Services/PrivateAIIntegrationService.swift
index 2b9dc2c4..24cf3629 100644
--- a/Sources/Fluid/Services/PrivateAIIntegrationService.swift
+++ b/Sources/Fluid/Services/PrivateAIIntegrationService.swift
@@ -237,8 +237,17 @@ actor PrivateAIIntegrationService {
await Self.provider.loadedModelState()
}
+ /// Experimental: gives the model's memory back after a quiet period. The
+ /// next dictation reloads it while the user is still speaking.
+ nonisolated static let idleUnloader = PrivateAIIdleUnloader(
+ delay: .seconds(600),
+ isEnabled: { await MainActor.run { SettingsStore.shared.privateAIIdleUnloadEnabled } },
+ isBusy: { await MainActor.run { AppServices.shared.asr.isRunningOrStarting } },
+ unload: { await PrivateAIIntegrationService.shared.unloadCachedRuntime(reason: "idle") }
+ )
+
func loadModel(_ model: PrivateAIRegisteredModel) async throws -> PrivateAIStatus {
- let status = try await Self.provider.loadModel(model)
+ let status = try await Self.idleUnloader.tracking { try await Self.provider.loadModel(model) }
guard status.state == .ready else { return status }
await self.removeInactiveInstalledModels(keeping: model)
@@ -246,7 +255,7 @@ actor PrivateAIIntegrationService {
}
func verifyModel(_ model: PrivateAIRegisteredModel) async throws -> PrivateAIStatus {
- try await Self.provider.verifyModel(model)
+ try await Self.idleUnloader.tracking { try await Self.provider.verifyModel(model) }
}
func removeInactiveInstalledModels(keeping model: PrivateAIRegisteredModel) async {
@@ -272,7 +281,7 @@ actor PrivateAIIntegrationService {
}
func prewarmDictation() async {
- await Self.provider.prewarmDictation()
+ await Self.idleUnloader.tracking { await Self.provider.prewarmDictation() }
}
func unloadCachedRuntime(reason: String = "manual") async {
@@ -289,12 +298,14 @@ actor PrivateAIIntegrationService {
context: AppContext
) async throws -> EnhancementResult {
let budget = try Self.validatedDictationBudget(inputText, contextTokenLimit: runtime.contextTokenLimit)
- return try await self.dictationProvider.enhanceDictation(
- inputText,
- runtime: runtime,
- context: context,
- maxOutputTokens: budget.maxOutputTokens
- )
+ return try await Self.idleUnloader.tracking {
+ try await self.dictationProvider.enhanceDictation(
+ inputText,
+ runtime: runtime,
+ context: context,
+ maxOutputTokens: budget.maxOutputTokens
+ )
+ }
}
nonisolated func enhanceDictation(
@@ -304,13 +315,15 @@ actor PrivateAIIntegrationService {
streamHandler: PrivateAIStreamHandler?
) async throws -> EnhancementResult {
let budget = try Self.validatedDictationBudget(inputText, contextTokenLimit: runtime.contextTokenLimit)
- return try await self.dictationProvider.enhanceDictation(
- inputText,
- runtime: runtime,
- context: context,
- maxOutputTokens: budget.maxOutputTokens,
- streamHandler: streamHandler
- )
+ return try await Self.idleUnloader.tracking {
+ try await self.dictationProvider.enhanceDictation(
+ inputText,
+ runtime: runtime,
+ context: context,
+ maxOutputTokens: budget.maxOutputTokens,
+ streamHandler: streamHandler
+ )
+ }
}
private nonisolated static func validatedDictationBudget(
@@ -333,12 +346,14 @@ actor PrivateAIIntegrationService {
runtime: RuntimeConfiguration,
context: AppContext
) async throws -> EnhancementResult {
- try await Self.provider.rewrite(
- inputText,
- systemPrompt: systemPrompt,
- runtime: runtime,
- context: context
- )
+ try await Self.idleUnloader.tracking {
+ try await Self.provider.rewrite(
+ inputText,
+ systemPrompt: systemPrompt,
+ runtime: runtime,
+ context: context
+ )
+ }
}
}
diff --git a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
index 271d5e27..e178cf49 100644
--- a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
+++ b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
@@ -999,6 +999,13 @@ extension AIEnhancementSettingsView {
)
}
+ private var privateAIIdleUnloadBinding: Binding<Bool> {
+ Binding(
+ get: { self.settings.privateAIIdleUnloadEnabled },
+ set: { self.settings.privateAIIdleUnloadEnabled = $0 }
+ )
+ }
+
private var privateAIBoostBinding: Binding<Bool> {
Binding(
get: { self.settings.privateAIBoostEnabled },
@@ -1914,6 +1921,11 @@ extension AIEnhancementSettingsView {
Toggle("Faster first result", isOn: self.privateAIPrefixCacheBinding)
.labelsHidden().toggleStyle(.switch).disabled(isBusy)
}
+ Divider()
+ FluidManagementRow(title: "Free memory when idle", detail: "Experimental. Unloads the model after 10 quiet minutes and reloads it as you start speaking.") {
+ Toggle("Free memory when idle", isOn: self.privateAIIdleUnloadBinding)
+ .labelsHidden().toggleStyle(.switch)
+ }
if self.privateAIShowsBoostRow {
Divider()
FluidManagementRow(title: "Faster results", detail: "Extra local acceleration. Uses more memory.") {
diff --git a/Tests/PrivateAIIdleUnloaderTests.swift b/Tests/PrivateAIIdleUnloaderTests.swift
new file mode 100644
index 00000000..4b579c25
--- /dev/null
+++ b/Tests/PrivateAIIdleUnloaderTests.swift
@@ -0,0 +1,112 @@
+import Foundation
+
+private actor Recorder {
+ private(set) var unloads = 0
+ var enabled = true
+ var busy = false
+ func noteUnload() { self.unloads += 1 }
+ func set(enabled: Bool) { self.enabled = enabled }
+ func set(busy: Bool) { self.busy = busy }
+}
+
+private func makeUnloader(_ recorder: Recorder, delayMs: Int = 60) -> PrivateAIIdleUnloader {
+ PrivateAIIdleUnloader(
+ delay: .milliseconds(delayMs),
+ isEnabled: { await recorder.enabled },
+ isBusy: { await recorder.busy },
+ unload: { await recorder.noteUnload() }
+ )
+}
+
+private func pause(_ ms: Int) async {
+ try? await Task.sleep(for: .milliseconds(ms))
+}
+
+private func expect(_ condition: Bool, _ message: String) {
+ guard condition else {
+ FileHandle.standardError.write(Data("FAIL: \(message)\n".utf8))
+ exit(1)
+ }
+ print("ok: \(message)")
+}
+
+@main
+enum PrivateAIIdleUnloaderTests {
+ static func main() async {
+ do {
+ let recorder = Recorder()
+ let unloader = makeUnloader(recorder)
+ await unloader.tracking {}
+ await pause(250)
+ let unloads = await recorder.unloads
+ expect(unloads == 1, "unloads once after the quiet period")
+ }
+ do {
+ let recorder = Recorder()
+ let unloader = makeUnloader(recorder, delayMs: 150)
+ for _ in 0 ..< 4 {
+ await unloader.tracking {}
+ await pause(60)
+ }
+ var unloads = await recorder.unloads
+ expect(unloads == 0, "activity keeps restarting the countdown")
+ await pause(400)
+ unloads = await recorder.unloads
+ expect(unloads == 1, "unloads after activity stops")
+ }
+ do {
+ let recorder = Recorder()
+ let unloader = makeUnloader(recorder)
+ await unloader.begin()
+ await pause(250)
+ var unloads = await recorder.unloads
+ expect(unloads == 0, "never unloads while a request is in flight")
+ await unloader.end()
+ await pause(250)
+ unloads = await recorder.unloads
+ expect(unloads == 1, "unloads once the request finishes")
+ }
+ do {
+ let recorder = Recorder()
+ let unloader = makeUnloader(recorder)
+ await unloader.begin()
+ await unloader.tracking {}
+ await pause(250)
+ let unloads = await recorder.unloads
+ expect(unloads == 0, "overlapping requests hold the model until the last ends")
+ await unloader.end()
+ }
+ do {
+ let recorder = Recorder()
+ await recorder.set(enabled: false)
+ let unloader = makeUnloader(recorder)
+ await unloader.tracking {}
+ await pause(250)
+ let unloads = await recorder.unloads
+ expect(unloads == 0, "does nothing when the setting is off")
+ }
+ do {
+ let recorder = Recorder()
+ await recorder.set(busy: true)
+ let unloader = makeUnloader(recorder)
+ await unloader.tracking {}
+ await pause(250)
+ var unloads = await recorder.unloads
+ expect(unloads == 0, "waits while a recording is in progress")
+ await recorder.set(busy: false)
+ await pause(250)
+ unloads = await recorder.unloads
+ expect(unloads == 1, "unloads after the recording ends")
+ }
+ do {
+ struct Boom: Error {}
+ let recorder = Recorder()
+ let unloader = makeUnloader(recorder)
+ do { try await unloader.tracking { throw Boom() } } catch {}
+ await pause(250)
+ let unloads = await recorder.unloads
+ expect(unloads == 1, "a failed request still releases its hold")
+ }
+ print("PrivateAIIdleUnloaderTests passed")
+ }
+}
diff --git a/Tests/run_private_ai_idle_unloader_tests.sh b/Tests/run_private_ai_idle_unloader_tests.sh
new file mode 100755
index 00000000..30e3aa7b
--- /dev/null
+++ b/Tests/run_private_ai_idle_unloader_tests.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+
+task_developer_dir="${DEVELOPER_DIR:-$(xcode-select -p)}"
+case "$task_developer_dir" in
+ */Xcode*.app/Contents/Developer) ;;
+ *) echo "Set DEVELOPER_DIR to an installed full Xcode before running tests." >&2; exit 1 ;;
+esac
+export DEVELOPER_DIR="$task_developer_dir"
+task_repo_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+task_test_dir=$(mktemp -d /tmp/fluidvoice-idle-unloader-tests.XXXXXX)
+xcrun swiftc -O -parse-as-library \
+ "$task_repo_dir/Sources/Fluid/Services/PrivateAIIdleUnloader.swift" \
+ "$task_repo_dir/Tests/PrivateAIIdleUnloaderTests.swift" \
+ -o "$task_test_dir/idle-unloader-tests"
+"$task_test_dir/idle-unloader-tests"