Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat(private-ai): choose the idle unload period

altic-devba1d2af68330
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index b1e055e9..67bf8f9d 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -1692,12 +1692,35 @@ final class SettingsStore: ObservableObject {
}
}
- /// Experimental. Off by default: the model stays loaded as before.
- var privateAIIdleUnloadEnabled: Bool {
- get { self.defaults.bool(forKey: Keys.privateAIIdleUnloadEnabled) }
+ /// How long the local model may sit unused before its memory is released.
+ enum PrivateAIIdleUnload: Int, CaseIterable, Identifiable {
+ case never = 0
+ case tenMinutes = 10
+ case thirtyMinutes = 30
+ case oneHour = 60
+
+ var id: Int { self.rawValue }
+
+ var title: String {
+ switch self {
+ case .never: "Never"
+ case .tenMinutes: "10 minutes"
+ case .thirtyMinutes: "30 minutes"
+ case .oneHour: "1 hour"
+ }
+ }
+
+ var delay: Duration? {
+ self == .never ? nil : .seconds(self.rawValue * 60)
+ }
+ }
+
+ /// Experimental. `.never` by default: the model stays loaded as before.
+ var privateAIIdleUnload: PrivateAIIdleUnload {
+ get { PrivateAIIdleUnload(rawValue: self.defaults.integer(forKey: Keys.privateAIIdleUnloadMinutes)) ?? .never }
set {
objectWillChange.send()
- self.defaults.set(newValue, forKey: Keys.privateAIIdleUnloadEnabled)
+ self.defaults.set(newValue.rawValue, forKey: Keys.privateAIIdleUnloadMinutes)
}
}
@@ -5657,7 +5680,7 @@ private extension SettingsStore {
static let selectedProviderID = "SelectedProviderID"
static let privateAIPrefixKVCacheEnabled = "PrivateAIProviderPrefixKVCacheEnabled"
static let privateAIBoostEnabled = "PrivateAIProviderBoostEnabled"
- static let privateAIIdleUnloadEnabled = "PrivateAIProviderIdleUnloadEnabled"
+ static let privateAIIdleUnloadMinutes = "PrivateAIProviderIdleUnloadMinutes"
static let privateAIBackendPreference = SettingsStore.privateAIBackendPreferenceDefaultsKey
static let privateAIContextTokenLimit = "PrivateAIProviderContextTokenLimit"
static let privateAIContextDefaultMigratedTo4K = "PrivateAIProviderContextDefaultMigratedTo4K"
diff --git a/Sources/Fluid/Services/PrivateAIIdleUnloader.swift b/Sources/Fluid/Services/PrivateAIIdleUnloader.swift
index c64134a0..184a0dfe 100644
--- a/Sources/Fluid/Services/PrivateAIIdleUnloader.swift
+++ b/Sources/Fluid/Services/PrivateAIIdleUnloader.swift
@@ -5,21 +5,19 @@ import Foundation
/// state the caller reports as busy (a recording that will need the model when
/// it ends), postpones the unload rather than cancelling it.
actor PrivateAIIdleUnloader {
- private let delay: Duration
- private let isEnabled: @Sendable () async -> Bool
+ /// The quiet period to wait, or nil while the feature is off.
+ private let delay: @Sendable () async -> Duration?
private let isBusy: @Sendable () async -> Bool
private let unload: @Sendable () async -> Void
private var inFlight = 0
private var pending: Task<Void, Never>?
init(
- delay: Duration,
- isEnabled: @escaping @Sendable () async -> Bool,
+ delay: @escaping @Sendable () async -> Duration?,
isBusy: @escaping @Sendable () async -> Bool,
unload: @escaping @Sendable () async -> Void
) {
self.delay = delay
- self.isEnabled = isEnabled
self.isBusy = isBusy
self.unload = unload
}
@@ -49,9 +47,17 @@ actor PrivateAIIdleUnloader {
}
}
+ /// Restarts the countdown with the current setting, e.g. after the user
+ /// picks a different quiet period.
+ func settingsChanged() {
+ guard self.inFlight == 0 else { return }
+ self.schedule()
+ }
+
private func schedule() {
self.pending?.cancel()
self.pending = Task { [delay] in
+ guard let delay = await delay() else { return }
try? await Task.sleep(for: delay)
guard !Task.isCancelled else { return }
await self.fire()
@@ -59,7 +65,7 @@ actor PrivateAIIdleUnloader {
}
private func fire() async {
- guard self.inFlight == 0, await self.isEnabled() else { return }
+ guard self.inFlight == 0, await self.delay() != nil else { return }
if await self.isBusy() {
self.schedule()
return
diff --git a/Sources/Fluid/Services/PrivateAIIntegrationService.swift b/Sources/Fluid/Services/PrivateAIIntegrationService.swift
index 24cf3629..d519dda9 100644
--- a/Sources/Fluid/Services/PrivateAIIntegrationService.swift
+++ b/Sources/Fluid/Services/PrivateAIIntegrationService.swift
@@ -240,8 +240,7 @@ actor PrivateAIIntegrationService {
/// Experimental: gives the model's memory back after a quiet period. The
/// next dictation reloads it while the user is still speaking.
nonisolated static let idleUnloader = PrivateAIIdleUnloader(
- delay: .seconds(600),
- isEnabled: { await MainActor.run { SettingsStore.shared.privateAIIdleUnloadEnabled } },
+ delay: { await MainActor.run { SettingsStore.shared.privateAIIdleUnload.delay } },
isBusy: { await MainActor.run { AppServices.shared.asr.isRunningOrStarting } },
unload: { await PrivateAIIntegrationService.shared.unloadCachedRuntime(reason: "idle") }
)
diff --git a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
index e178cf49..d3ca9e56 100644
--- a/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
+++ b/Sources/Fluid/UI/AISettingsView+AIConfiguration.swift
@@ -999,10 +999,13 @@ extension AIEnhancementSettingsView {
)
}
- private var privateAIIdleUnloadBinding: Binding<Bool> {
+ private var privateAIIdleUnloadBinding: Binding<SettingsStore.PrivateAIIdleUnload> {
Binding(
- get: { self.settings.privateAIIdleUnloadEnabled },
- set: { self.settings.privateAIIdleUnloadEnabled = $0 }
+ get: { self.settings.privateAIIdleUnload },
+ set: { option in
+ self.settings.privateAIIdleUnload = option
+ Task { await PrivateAIIntegrationService.idleUnloader.settingsChanged() }
+ }
)
}
@@ -1922,9 +1925,13 @@ extension AIEnhancementSettingsView {
.labelsHidden().toggleStyle(.switch).disabled(isBusy)
}
Divider()
- FluidManagementRow(title: "Free memory when idle", detail: "Experimental. Unloads the model after 10 quiet minutes and reloads it as you start speaking.") {
- Toggle("Free memory when idle", isOn: self.privateAIIdleUnloadBinding)
- .labelsHidden().toggleStyle(.switch)
+ FluidManagementRow(title: "Free memory when idle", detail: "Experimental. Unloads the model after a quiet period and reloads it as you start speaking.") {
+ Picker("Free memory when idle", selection: self.privateAIIdleUnloadBinding) {
+ ForEach(SettingsStore.PrivateAIIdleUnload.allCases) { option in
+ Text(option.title).tag(option)
+ }
+ }
+ .labelsHidden().frame(width: 200)
}
if self.privateAIShowsBoostRow {
Divider()
diff --git a/Tests/PrivateAIIdleUnloaderTests.swift b/Tests/PrivateAIIdleUnloaderTests.swift
index 4b579c25..5a22a033 100644
--- a/Tests/PrivateAIIdleUnloaderTests.swift
+++ b/Tests/PrivateAIIdleUnloaderTests.swift
@@ -2,17 +2,16 @@ import Foundation
private actor Recorder {
private(set) var unloads = 0
- var enabled = true
+ var delayMs: Int? = 60
var busy = false
func noteUnload() { self.unloads += 1 }
- func set(enabled: Bool) { self.enabled = enabled }
+ func set(delayMs: Int?) { self.delayMs = delayMs }
func set(busy: Bool) { self.busy = busy }
}
-private func makeUnloader(_ recorder: Recorder, delayMs: Int = 60) -> PrivateAIIdleUnloader {
+private func makeUnloader(_ recorder: Recorder) -> PrivateAIIdleUnloader {
PrivateAIIdleUnloader(
- delay: .milliseconds(delayMs),
- isEnabled: { await recorder.enabled },
+ delay: { await recorder.delayMs.map { .milliseconds($0) } },
isBusy: { await recorder.busy },
unload: { await recorder.noteUnload() }
)
@@ -43,7 +42,8 @@ enum PrivateAIIdleUnloaderTests {
}
do {
let recorder = Recorder()
- let unloader = makeUnloader(recorder, delayMs: 150)
+ await recorder.set(delayMs: 150)
+ let unloader = makeUnloader(recorder)
for _ in 0 ..< 4 {
await unloader.tracking {}
await pause(60)
@@ -78,12 +78,17 @@ enum PrivateAIIdleUnloaderTests {
}
do {
let recorder = Recorder()
- await recorder.set(enabled: false)
+ await recorder.set(delayMs: nil)
let unloader = makeUnloader(recorder)
await unloader.tracking {}
await pause(250)
- let unloads = await recorder.unloads
- expect(unloads == 0, "does nothing when the setting is off")
+ var unloads = await recorder.unloads
+ expect(unloads == 0, "does nothing when set to never")
+ await recorder.set(delayMs: 60)
+ await unloader.settingsChanged()
+ await pause(250)
+ unloads = await recorder.unloads
+ expect(unloads == 1, "picking a period starts the countdown without new activity")
}
do {
let recorder = Recorder()