Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Fix mobile warmup recovery and picker retention

setalp9bea106f91ec
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 7b57bd60eb..4343093ab7 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -488,6 +488,7 @@ internal class ChatStoreActions(
private fun startGeneration(sessionId: String, userMessage: ChatMessage) {
val scope = scope ?: return
if (!state.value.chat.deviceCapability.isChatSupported()) return
+ modelSettingsActions.cancelChatWarmup(releaseLoadedModel = false)
val prompt = buildPrompt(userMessage.text, userMessage.attachments)
val priorGeneration = generationJob
val priorSummary = sessionSummaryJob
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index e912d6000c..1e8846c690 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -138,10 +138,13 @@ class LlmProvider(
}
}
- private suspend fun <T> withResources(block: suspend () -> T): T {
+ private suspend fun <T> withResources(
+ retainAfterCooldown: Boolean = false,
+ block: suspend () -> T,
+ ): T {
var releaseAfterUse = false
return withModelResources(modelLoadMutex, { releaseIdleResourcesLocked(releaseAfterUse) }) {
- releaseAfterUse = memoryPressure.suppressesOptionalWork()
+ releaseAfterUse = !retainAfterCooldown && memoryPressure.suppressesOptionalWork()
block()
}
}
@@ -151,13 +154,14 @@ class LlmProvider(
currentCoroutineContext().ensureActive()
if (!chatForeground || memoryPressure.suppressesOptionalWork()) return@withResources
prepareVoiceResources()
+ var completed = false
try {
transcriber.loadModel()
currentCoroutineContext().ensureActive()
voiceOwner = owner
- } catch (error: Throwable) {
- unloadTranscriptionModelIfLoaded()
- throw error
+ completed = true
+ } finally {
+ if (!completed) unloadTranscriptionModelIfLoaded()
}
}
}
@@ -225,29 +229,54 @@ class LlmProvider(
internal suspend fun prewarmChatModelIfDownloaded(
selection: LlmModelSelection,
owner: String,
- ): Unit = withModelContext {
- withResources {
- currentCoroutineContext().ensureActive()
- if (
- !chatForeground ||
- !appForeground ||
+ ): ChatWarmupResult = withModelContext {
+ val result =
+ withResources(retainAfterCooldown = true) {
+ currentCoroutineContext().ensureActive()
+ if (
+ !chatForeground ||
+ !appForeground ||
+ voiceOwner != null ||
+ !isChatModelReady(selection)
+ )
+ return@withResources ChatWarmupResult.Skipped
+ val cooldown = memoryPressure.remainingCooldownMillis()
+ if (cooldown > 0) return@withResources ChatWarmupResult.RetryAfter(cooldown)
+ if (
memoryPressure.suppressesOptionalWork() ||
- deviceCapabilityProvider.isLowMemory() ||
- voiceOwner != null ||
- !isChatModelReady(selection)
- )
- return@withResources
- unloadTranscriptionModelIfLoaded()
- currentCoroutineContext().ensureActive()
- if (loadedContextLength(selection) != null) return@withResources
- try {
- ensureModelReadyLocked(selection, {}, allowRecovery = false, shouldDownload = false)
+ deviceCapabilityProvider.isLowMemory()
+ )
+ return@withResources ChatWarmupResult.Skipped
+ unloadTranscriptionModelIfLoaded()
currentCoroutineContext().ensureActive()
- chatWarmupOwner = owner
- } catch (error: Throwable) {
- unloadModel()
- throw error
+ if (loadedContextLength(selection) != null) {
+ if (chatWarmupOwner != null) chatWarmupOwner = owner
+ return@withResources ChatWarmupResult.Ready
+ }
+ var completed = false
+ try {
+ ensureModelReadyLocked(
+ selection,
+ {},
+ allowRecovery = false,
+ shouldDownload = false,
+ )
+ currentCoroutineContext().ensureActive()
+ chatWarmupOwner = owner
+ completed = true
+ } finally {
+ if (!completed) unloadModel()
+ }
+ ChatWarmupResult.Ready
}
+ if (result != ChatWarmupResult.Ready) return@withModelContext result
+ val cooldown = memoryPressure.remainingCooldownMillis()
+ when {
+ !appForeground -> ChatWarmupResult.Skipped
+ cooldown > 0 -> ChatWarmupResult.RetryAfter(cooldown)
+ !memoryPressure.suppressesOptionalWork() && loadedContextLength(selection) != null ->
+ ChatWarmupResult.Ready
+ else -> ChatWarmupResult.Skipped
}
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelMemoryPressure.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelMemoryPressure.kt
index 62e935f6df..9e7e61bfbf 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelMemoryPressure.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelMemoryPressure.kt
@@ -1,5 +1,8 @@
package io.ente.ensu.llm
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
@@ -18,6 +21,9 @@ internal class ModelMemoryPressure(private val elapsedRealtime: () -> Long) {
fun suppressesOptionalWork(): Boolean =
requestedGeneration != releasedGeneration || elapsedRealtime() < retryAfter
+ @Synchronized
+ fun remainingCooldownMillis(): Long = (retryAfter - elapsedRealtime()).coerceAtLeast(0)
+
@Synchronized
fun pendingEviction(): Long? = requestedGeneration.takeIf { it != releasedGeneration }
@@ -31,6 +37,35 @@ internal class ModelMemoryPressure(private val elapsedRealtime: () -> Long) {
}
}
+internal sealed interface ChatWarmupResult {
+ data object Ready : ChatWarmupResult
+
+ data object Skipped : ChatWarmupResult
+
+ data class RetryAfter(val millis: Long) : ChatWarmupResult
+}
+
+internal suspend fun retryChatWarmup(
+ isEligible: () -> Boolean,
+ wait: suspend (Long) -> Unit = { delay(it) },
+ operation: suspend () -> ChatWarmupResult,
+): Boolean {
+ while (true) {
+ currentCoroutineContext().ensureActive()
+ if (!isEligible()) return false
+ val result = operation()
+ currentCoroutineContext().ensureActive()
+ when (result) {
+ ChatWarmupResult.Ready -> return true
+ ChatWarmupResult.Skipped -> return false
+ is ChatWarmupResult.RetryAfter -> {
+ if (result.millis <= 0) return false
+ wait(result.millis)
+ }
+ }
+ }
+}
+
internal suspend fun <T> withModelResources(
mutex: Mutex,
releaseIfNeeded: () -> Unit,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index 0570f76e72..bf6b8e42c4 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -360,7 +360,6 @@ internal class ModelSettingsActions(
fun setPickerPending(pending: Boolean) {
pickerPending = pending
- if (!pending && !appForeground) setAppForeground(false)
}
private fun releaseBackgroundResources() {
@@ -382,7 +381,7 @@ internal class ModelSettingsActions(
}
}
- private fun cancelChatWarmup(releaseLoadedModel: Boolean = true) {
+ fun cancelChatWarmup(releaseLoadedModel: Boolean = true) {
warmupJob?.cancel()
warmupJob = null
val owner = warmupOwner
@@ -417,23 +416,29 @@ internal class ModelSettingsActions(
warmupOwner = owner
warmupSelection = selection
warmupJob = scope.launch {
+ var ready = false
try {
delay(350)
awaitKnowledgeReady()
notesStore.awaitReady()
coroutineContext.ensureActive()
- val latest = state.value
- if (
- !appForeground ||
- !chatActive ||
- activeVoiceJobs.get() > 0 ||
- latest.chat.isGenerating ||
- latest.chat.isDownloading
- ) {
- if (warmupOwner == owner) cancelChatWarmup()
- return@launch
- }
- llmProvider.prewarmChatModelIfDownloaded(selection, owner)
+ ready =
+ retryChatWarmup(
+ isEligible = {
+ val latest = state.value
+ warmupOwner == owner &&
+ resolveSelection(latest.modelSettings) == selection &&
+ appForeground &&
+ chatActive &&
+ !warmupSuppressed &&
+ activeVoiceJobs.get() == 0 &&
+ latest.chat.deviceCapability.isChatSupported() &&
+ !latest.chat.isGenerating &&
+ !latest.chat.isDownloading
+ }
+ ) {
+ llmProvider.prewarmChatModelIfDownloaded(selection, owner)
+ }
} catch (error: CancellationException) {
throw error
} catch (error: Exception) {
@@ -443,6 +448,14 @@ internal class ModelSettingsActions(
tag = "Model",
throwable = error,
)
+ } finally {
+ if (warmupOwner == owner) {
+ warmupJob = null
+ if (!ready) {
+ warmupOwner = null
+ warmupSelection = null
+ }
+ }
}
}
}
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
index 82db127727..6c422882f2 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
@@ -1,5 +1,6 @@
package io.ente.ensu.llm
+import io.ente.ensu.coroutines.runCatchingCancellable
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
@@ -23,7 +24,7 @@ class ModelMemoryPressureTest {
var now = 0L
val pressure = ModelMemoryPressure { now }
pressure.requestEviction()
- val eviction = pressure.pendingEviction()!!
+ val eviction = checkNotNull(pressure.pendingEviction())
now = 60_000
assertEquals(eviction, pressure.pendingEviction())
@@ -33,7 +34,7 @@ class ModelMemoryPressureTest {
assertNull(pressure.pendingEviction())
assertFalse(pressure.suppressesOptionalWork())
pressure.requestEviction()
- pressure.didRelease(pressure.pendingEviction()!!)
+ pressure.didRelease(checkNotNull(pressure.pendingEviction()))
now = 89_999
assertTrue(pressure.suppressesOptionalWork())
@@ -46,10 +47,10 @@ class ModelMemoryPressureTest {
var now = 0L
val pressure = ModelMemoryPressure { now }
pressure.requestEviction()
- val earlier = pressure.pendingEviction()!!
+ val earlier = checkNotNull(pressure.pendingEviction())
now = 20_000
pressure.requestEviction()
- val later = pressure.pendingEviction()!!
+ val later = checkNotNull(pressure.pendingEviction())
pressure.didRelease(earlier)
assertEquals(later, pressure.pendingEviction())
@@ -133,7 +134,7 @@ class ModelMemoryPressureTest {
fun failedWorkReleasesResourcesBeforeUnlocking() = runBlocking {
val mutex = Mutex()
var resident = false
- val result = runCatching {
+ val result = runCatchingCancellable {
withModelResources(mutex, { resident = false }) {
resident = true
error("Native operation failed")
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 7cbc4e91ae..2d1d8f92f5 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -287,22 +287,28 @@ final class ChatViewModel: ObservableObject {
warmupAttempt = (owner, key)
warmupTask = Task { [weak self] in
guard let self else { return }
+ var ready = false
defer {
- if warmupAttempt?.owner == owner { warmupTask = nil }
+ if warmupAttempt?.owner == owner {
+ warmupTask = nil
+ if !ready { warmupAttempt = nil }
+ }
}
do {
try await Task.sleep(nanoseconds: 350_000_000)
await knowledgeStore.bootstrap()
await notesStore.bootstrap()
- try Task.checkCancellation()
- guard appForeground, chatActive, !isGenerating, !isDownloading,
- !voiceInputState.isWorking,
- !voiceTranscriber.hasActiveTasks
- else {
- if warmupAttempt?.owner == owner { cancelChatWarmup() }
- return
- }
- try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)
+ ready = try await retryChatWarmup(
+ isEligible: {
+ warmupAttempt?.owner == owner
+ && modelReadyKey(for: modelSettings.currentSelection()) == key
+ && appForeground && chatActive && !warmupSuppressed
+ && !isChatUnsupported && !isGenerating && !isDownloading
+ && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks
+ },
+ operation: {
+ try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)
+ })
} catch {
guard !Task.isCancelled, !isCancellation(error), warmupAttempt?.owner == owner,
modelReadyKey(for: modelSettings.currentSelection()) == key,
@@ -1037,6 +1043,7 @@ final class ChatViewModel: ObservableObject {
try await self.ensureRequiredModelsReadyShared(
selection, recoverDownloadedModel: wasDownloaded)
self.handleProgress(DownloadProgress(percent: 100, status: "Ready", phase: .ready))
+ self.refreshChatWarmup()
} catch {
if isCancellation(error) {
return
@@ -1167,6 +1174,7 @@ final class ChatViewModel: ObservableObject {
showUnsupportedDeviceDialog = true
return
}
+ cancelChatWarmup(releaseLoadedModel: false)
let selection = modelSettings.currentSelection()
let prompt = buildPrompt(text: userNode.text, attachments: userNode.attachments)
let priorGeneration = generationTask
diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index a7afec916f..54da640188 100644
--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -203,6 +203,7 @@ actor LlmProvider {
private func withModelLock<T>(
isolation: isolated (any Actor)? = #isolation,
suspendMaintenance: Bool = true,
+ retainAfterCooldown: Bool = false,
_ operation: () async throws -> T
) async throws -> T {
var scope: ModelUseScope?
@@ -215,7 +216,9 @@ actor LlmProvider {
return try await modelLoadGate.withLock(
onEnter: {
await self.reconcileIdleResources()
- releaseAfterUse = await self.memoryPressureSuppressed
+ if !retainAfterCooldown {
+ releaseAfterUse = await self.memoryPressureSuppressed
+ }
},
onExit: { await self.reconcileIdleResources(force: releaseAfterUse) }, operation)
}
@@ -325,14 +328,21 @@ actor LlmProvider {
func prewarmChatModelIfDownloaded(
_ selection: LlmModelSelection, owner: UUID
- ) async throws {
- try await withModelLock {
- guard appForeground, chatForeground, !memoryPressureSuppressed, voiceOwner == nil,
+ ) async throws -> ChatWarmupResult {
+ let result: ChatWarmupResult = try await withModelLock(retainAfterCooldown: true) {
+ guard appForeground, chatForeground, voiceOwner == nil,
isChatModelReady(selection)
- else { return }
+ else { return .skipped }
+ let cooldown = memoryPressure.remainingCooldown(
+ now: ProcessInfo.processInfo.systemUptime)
+ if cooldown > 0 { return .retryAfter(cooldown) }
+ guard !memoryPressureSuppressed else { return .skipped }
unloadTranscriptionModelIfLoaded()
try Task.checkCancellation()
- guard loadedContextLength(selection) == nil else { return }
+ guard loadedContextLength(selection) == nil else {
+ if chatWarmupOwner != nil { chatWarmupOwner = owner }
+ return .ready
+ }
do {
try await ensureModelReadyLocked(
selection, onProgress: { _ in }, allowRecovery: false, shouldDownload: false)
@@ -342,7 +352,14 @@ actor LlmProvider {
unloadModel()
throw error
}
- }
+ return .ready
+ }
+ guard case .ready = result else { return result }
+ guard appForeground else { return .skipped }
+ let cooldown = memoryPressure.remainingCooldown(now: ProcessInfo.processInfo.systemUptime)
+ if cooldown > 0 { return .retryAfter(cooldown) }
+ return !memoryPressureSuppressed && loadedContextLength(selection) != nil
+ ? .ready : .skipped
}
func releaseChatWarmup(owner: UUID) async {
diff --git a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
index cba7099363..8742f8b2a3 100644
--- a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
+++ b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
@@ -14,6 +14,10 @@ struct ModelMemoryPressureState {
cleanupPending || now < retryAfter
}
+ func remainingCooldown(now: TimeInterval) -> TimeInterval {
+ max(0, retryAfter - now)
+ }
+
func requiresCleanup(appForeground: Bool, now: TimeInterval) -> Bool {
!appForeground || suppressesOptionalWork(now: now)
}
@@ -23,6 +27,37 @@ struct ModelMemoryPressureState {
}
}
+enum ChatWarmupResult {
+ case ready
+ case skipped
+ case retryAfter(TimeInterval)
+}
+
+func retryChatWarmup(
+ isolation: isolated (any Actor)? = #isolation,
+ isEligible: () -> Bool,
+ wait: (TimeInterval) async throws -> Void = {
+ try await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000))
+ },
+ operation: () async throws -> ChatWarmupResult
+) async throws -> Bool {
+ while true {
+ try Task.checkCancellation()
+ guard isEligible() else { return false }
+ let result = try await operation()
+ try Task.checkCancellation()
+ switch result {
+ case .ready:
+ return true
+ case .skipped:
+ return false
+ case .retryAfter(let delay):
+ guard delay > 0 else { return false }
+ try await wait(delay)
+ }
+ }
+}
+
actor AsyncSerialGate {
private var isLocked = false
private var waiters: [CheckedContinuation<Void, Never>] = []