π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Refine mobile model memory coordination
setalp715916d7ebf6
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex e3d55bea77..7b57bd60eb 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -616,6 +616,7 @@ internal class ChatStoreActions(} catch (error: kotlinx.coroutines.CancellationException) {throw error} catch (error: ModelMemoryUnavailable) {+ if (!isActive() || stopRequested) return@launchstate.update {it.copy(chat =@@ -761,20 +762,18 @@ internal class ChatStoreActions(onModelProgress,{ isActive() && state.value.modelSettings == settings },)- if (embeddingAssetInvalid) modelSettingsActions.refreshModelDownloadInfo()- return@launch+ } else {+ generateImageConversation(+ sessionId,+ userMessage,+ prompt,+ selection,+ modelSettingsActions.resolveTemperature(settings),+ knowledgeHits,+ onModelProgress,+ { isActive() && state.value.modelSettings == settings },+ )}-- generateImageConversation(- sessionId,- userMessage,- prompt,- selection,- modelSettingsActions.resolveTemperature(settings),- knowledgeHits,- onModelProgress,- { isActive() && state.value.modelSettings == settings },- )if (embeddingAssetInvalid) modelSettingsActions.refreshModelDownloadInfo()}generationJob = activeJobdiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex c0caaae14d..1aa2754e29 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -21,10 +21,7 @@ import io.ente.ensu.bindings.TranscriptionExceptionimport io.ente.ensu.bindings.transcriptionModelAssetimport io.ente.ensu.bindings.voiceActivityModelAssetimport io.ente.ensu.llm.LlmProvider-import io.ente.ensu.llm.ModelMaintenanceimport io.ente.ensu.llm.ModelMemoryUnavailable-import io.ente.ensu.llm.withMaintenanceSuspended-import io.ente.ensu.notes.LocalNotesStoreimport java.io.ByteArrayOutputStreamimport java.io.IOExceptionimport java.util.UUID@@ -90,14 +87,12 @@ internal fun rememberVoiceTranscriptionController(onVoiceInputJob: (Job) -> Unit,): VoiceTranscriptionController {val lifecycleOwner = LocalLifecycleOwner.current- val notes = LocalNotesStore.currentval controller =- remember(llmProvider, notes) {+ remember(llmProvider) {VoiceTranscriptionController(assetStore,llmProvider,onTranscript,- notes,onVoiceInputJob,)}@@ -118,7 +113,6 @@ internal class VoiceTranscriptionController(private val assetStore: AssetStore,private val llmProvider: LlmProvider,private val onTranscript: (String) -> Unit,- private val maintenance: ModelMaintenance?,private val onVoiceInputJob: (Job) -> Unit,) {private val modelAssets =@@ -309,77 +303,65 @@ internal class VoiceTranscriptionController(audioRecord = null}- private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) =- maintenance.withMaintenanceSuspended {- val owner = voiceOwner- try {- ensureTranscriptionModelDownloaded()- transcriptionPreloadJob?.join()- coroutineContext.ensureActive()- val transcript =- withContext(Dispatchers.IO) {- withContext(Dispatchers.Main.immediate) {- state = VoiceInputState.Transcribing- }- llmProvider.transcribe(- sampleRate.toUInt(),- pcm,- )- }- .trim()-- if (transcript.isBlank()) {- showTransientError("No speech detected.")- } else {- onTranscript(transcript)- state = VoiceInputState.Idle- }- } catch (error: CancellationException) {- throw error- } catch (error: UnsatisfiedLinkError) {- Log.w(TAG, "Voice transcription is not available on this device", error)- state =- VoiceInputState.Error("Voice transcription is not available on this device.")- } catch (error: TranscriptionException) {- Log.w(TAG, "Voice transcription failed: ${error.message}", error)- state = VoiceInputState.Error(transcriptionErrorMessage(error))- } catch (error: ModelMemoryUnavailable) {- state = VoiceInputState.Error(checkNotNull(error.message))- } catch (_: AssetDownloadException.Cancelled) {- state = VoiceInputState.Idle- } catch (error: AssetDownloadException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (_: LlmException.Cancelled) {+ private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) {+ val owner = voiceOwner+ try {+ ensureTranscriptionModelDownloaded()+ transcriptionPreloadJob?.join()+ coroutineContext.ensureActive()+ state = VoiceInputState.Transcribing+ val transcript = llmProvider.transcribe(sampleRate.toUInt(), pcm).trim()++ if (transcript.isBlank()) {+ showTransientError("No speech detected.")+ } else {+ onTranscript(transcript)state = VoiceInputState.Idle- } catch (error: LlmException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (error: Throwable) {- Log.w(- TAG,- "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",- error,- )- state = VoiceInputState.Error("Could not transcribe voice input.")- } finally {- withContext(NonCancellable) { llmProvider.releaseVoice(owner) }}+ } catch (error: CancellationException) {+ throw error+ } catch (error: UnsatisfiedLinkError) {+ Log.w(TAG, "Voice transcription is not available on this device", error)+ state = VoiceInputState.Error("Voice transcription is not available on this device.")+ } catch (error: TranscriptionException) {+ Log.w(TAG, "Voice transcription failed: ${error.message}", error)+ state = VoiceInputState.Error(transcriptionErrorMessage(error))+ } catch (error: ModelMemoryUnavailable) {+ state = VoiceInputState.Error(checkNotNull(error.message))+ } catch (_: AssetDownloadException.Cancelled) {+ state = VoiceInputState.Idle+ } catch (error: AssetDownloadException) {+ Log.w(TAG, "Voice model download failed: ${error.message}", error)+ state = VoiceInputState.Error(downloadErrorMessage)+ } catch (_: LlmException.Cancelled) {+ state = VoiceInputState.Idle+ } catch (error: LlmException) {+ Log.w(TAG, "Voice model download failed: ${error.message}", error)+ state = VoiceInputState.Error(downloadErrorMessage)+ } catch (error: Throwable) {+ Log.w(+ TAG,+ "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",+ error,+ )+ state = VoiceInputState.Error("Could not transcribe voice input.")+ } finally {+ withContext(NonCancellable) { llmProvider.releaseVoice(owner) }}+ }private fun preloadTranscriptionModel() {voiceOwner = UUID.randomUUID().toString()val owner = voiceOwner- transcriptionPreloadJob =- launchVoiceWork(Dispatchers.IO) {- try {- llmProvider.prewarmVoice(owner)- } catch (error: CancellationException) {- throw error- } catch (error: Throwable) {- Log.d(TAG, "Voice preload skipped", error)- }+ transcriptionPreloadJob = launchVoiceWork {+ try {+ llmProvider.prewarmVoice(owner)+ } catch (error: CancellationException) {+ throw error+ } catch (error: Throwable) {+ Log.d(TAG, "Voice preload skipped", error)}+ }}private fun releaseVoice() {@@ -387,7 +369,7 @@ internal class VoiceTranscriptionController(transcriptionPreloadJob = nullval owner = voiceOwnervoiceOwner = UUID.randomUUID().toString()- launchVoiceWork(NonCancellable + Dispatchers.IO) { llmProvider.releaseVoice(owner) }+ launchVoiceWork(NonCancellable) { llmProvider.releaseVoice(owner) }}private suspend fun ensureTranscriptionModelDownloaded() {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.ktindex 3d35a39aa5..46924626a9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt@@ -20,18 +20,14 @@ class AndroidDeviceCapabilityProvider(context: Context) {val manager = appContext.getSystemService(ActivityManager::class.java) ?: return nullval info = ActivityManager.MemoryInfo()manager.getMemoryInfo(info)- val process = android.os.Debug.MemoryInfo()- android.os.Debug.getMemoryInfo(process)- val processBytes = process.totalPss.toLong() * 1024return when {info.lowMemory -> 0uL- info.totalMem <= 0 || info.availMem < 0 || process.totalPss <= 0 -> null+ info.totalMem <= 0 || info.availMem < 0 -> nullelse ->modelMemoryHeadroomBytes(info.totalMem,info.availMem,info.threshold,- processBytes,optionalWork,optionalReserveCapBytes,)@@ -57,7 +53,6 @@ internal fun modelMemoryHeadroomBytes(totalBytes: Long,availableBytes: Long,lowMemoryThreshold: Long,- processBytes: Long,optionalWork: Boolean = true,optionalReserveCapBytes: Long? = null,): ULong {@@ -67,7 +62,5 @@ internal fun modelMemoryHeadroomBytes(if (optionalWork) minOf(totalBytes / 10, optionalReserveCapBytes ?: Long.MAX_VALUE)else 0,)- return minOf(availableBytes - systemReserve, totalBytes / 2 - processBytes)- .coerceAtLeast(0)- .toULong()+ return (availableBytes - systemReserve).coerceAtLeast(0).toULong()}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex c26d656e58..e912d6000c 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -103,10 +103,10 @@ class LlmProvider(return headroom != null && headroom >= budget.requiredBytes}- private fun hasRequiredWorkReserve(): Boolean {- val headroom = deviceCapabilityProvider.modelMemoryHeadroom(optionalWork = false)- return llmHasRequiredWorkReserve(headroom)- }+ private fun hasRequiredWorkReserve(): Boolean =+ llmHasRequiredWorkReserve(+ deviceCapabilityProvider.modelMemoryHeadroom(optionalWork = false)+ )private fun releaseProjector() {loadedContext?.releaseMultimodal()@@ -309,13 +309,7 @@ class LlmProvider(): GenerationSummary {deviceCapabilityProvider.chatCapability().requireChatSupported()currentJobId = null- unloadTranscriptionModelIfLoaded()- val mmprojPath =- if (imageFiles.isEmpty()) {- null- } else {- assetStore.llmMmprojPath(chatAsset(selection))?.absolutePath- }+ val mmprojPath = assetStore.llmMmprojPath(chatAsset(selection))?.absolutePathif (mmprojPath != null && !hasRequiredWorkReserve()) {throw ModelMemoryUnavailable()}@@ -362,12 +356,9 @@ class LlmProvider(coroutine.ensureActive()val model = checkNotNull(loadedModel) { "Model not loaded" }model- .newContext(- LlmContextParams(- contextSize = minOf(1024, checkNotNull(currentContextLength)),- nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),- nBatch = 128,- )+ .newTitleContext(+ chatContextSize = checkNotNull(currentContextLength).toUInt(),+ nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),).use { context ->coroutine.ensureActive()@@ -403,11 +394,7 @@ class LlmProvider(ensureModelReadyLocked(selection, onProgress)currentCoroutineContext().ensureActive()unloadTranscriptionModelIfLoaded()- try {- block(requireNotNull(loadedContext))- } finally {- releaseProjector()- }+ block(requireNotNull(loadedContext))}}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex bd974b29a4..0570f76e72 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -22,7 +22,6 @@ import kotlinx.coroutines.Jobimport kotlinx.coroutines.delayimport kotlinx.coroutines.ensureActiveimport kotlinx.coroutines.flow.MutableStateFlow-import kotlinx.coroutines.flow.combineimport kotlinx.coroutines.flow.updateimport kotlinx.coroutines.launch@@ -53,7 +52,7 @@ internal class ModelSettingsActions(llmProvider.setAppForeground(appForeground)awaitKnowledgeReady()notesStore.awaitReady()- combine(state, notesStore.state) { _, _ -> Unit }.collect { refreshChatWarmup() }+ state.collect { refreshChatWarmup() }}}diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.ktindex 78d4e41035..7ecc628284 100644--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt@@ -5,42 +5,32 @@ import org.junit.Testclass ModelMemoryHeadroomTest {@Test- fun profileReserveRespectsPlatformAndProcessLimits() {+ fun profileReserveRespectsPlatformThreshold() {assertEquals(1500uL,- modelMemoryHeadroomBytes(12000, 2000, 200, 200, optionalReserveCapBytes = 500),+ modelMemoryHeadroomBytes(12000, 2000, 200, optionalReserveCapBytes = 500),)assertEquals(0uL,- modelMemoryHeadroomBytes(12000, 400, 200, 200, optionalReserveCapBytes = 500),+ modelMemoryHeadroomBytes(12000, 400, 200, optionalReserveCapBytes = 500),)assertEquals(1200uL,- modelMemoryHeadroomBytes(12000, 2000, 800, 200, optionalReserveCapBytes = 500),+ modelMemoryHeadroomBytes(12000, 2000, 800, optionalReserveCapBytes = 500),)- assertEquals(- 100uL,- modelMemoryHeadroomBytes(12000, 2000, 200, 5900, optionalReserveCapBytes = 500),- )- }-- @Test- fun systemPressureLimitsAnOtherwiseSmallProcess() {- assertEquals(100uL, modelMemoryHeadroomBytes(4000, 500, 100, 100))- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 300, 100, 100))- assertEquals(100uL, modelMemoryHeadroomBytes(4000, 800, 700, 100))}@Test- fun nativeProcessFootprintLimitsAnOtherwiseIdleSystem() {- assertEquals(200uL, modelMemoryHeadroomBytes(4000, 3500, 100, 1800))- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 3500, 100, 2100))+ fun optionalHeadroomPreservesSystemReserve() {+ assertEquals(100uL, modelMemoryHeadroomBytes(4000, 500, 100))+ assertEquals(0uL, modelMemoryHeadroomBytes(4000, 300, 100))+ assertEquals(100uL, modelMemoryHeadroomBytes(4000, 800, 700))+ assertEquals(3100uL, modelMemoryHeadroomBytes(4000, 3500, 100))}@Test- fun requiredWorkPreservesPlatformAndProcessLimitsWithoutTheSpeculativeReserve() {- assertEquals(400uL, modelMemoryHeadroomBytes(4000, 500, 100, 100, optionalWork = false))- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 90, 100, 100, optionalWork = false))- assertEquals(200uL, modelMemoryHeadroomBytes(4000, 3500, 100, 1800, optionalWork = false))+ fun requiredWorkPreservesPlatformThresholdWithoutTheSpeculativeReserve() {+ assertEquals(400uL, modelMemoryHeadroomBytes(4000, 500, 100, optionalWork = false))+ assertEquals(0uL, modelMemoryHeadroomBytes(4000, 90, 100, optionalWork = false))}}diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.ktindex 1f581f545a..82db127727 100644--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt@@ -19,7 +19,7 @@ import org.junit.Testclass ModelMemoryPressureTest {@Test- fun evictionSurvivesTheAdmissionCooldownUntilResourcesAreReleased() {+ fun optionalWorkWaitsForBothReleaseAndCooldown() {var now = 0Lval pressure = ModelMemoryPressure { now }pressure.requestEviction()@@ -32,18 +32,12 @@ class ModelMemoryPressureTest {pressure.didRelease(eviction)assertNull(pressure.pendingEviction())assertFalse(pressure.suppressesOptionalWork())- }-- @Test- fun releaseDoesNotBypassCooldown() {- var now = 0L- val pressure = ModelMemoryPressure { now }pressure.requestEviction()pressure.didRelease(pressure.pendingEviction()!!)- now = 29_999+ now = 89_999assertTrue(pressure.suppressesOptionalWork())- now = 30_000+ now = 90_000assertFalse(pressure.suppressesOptionalWork())}@@ -87,10 +81,11 @@ class ModelMemoryPressureTest {}}releaseRequested = true- val cleanup = launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { release() } }+ val follower =+ launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { assertFalse(resident) } }mutex.unlock()operation.join()- cleanup.join()+ follower.join()assertEquals(listOf("release", "allocate", "release"), events)assertFalse(resident)@@ -117,14 +112,16 @@ class ModelMemoryPressureTest {entered.await()releaseRequested.set(true)operation.cancel()- val cleanup =- launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { release() } }+ val follower =+ launch(start = CoroutineStart.UNDISPATCHED) {+ mutex.withLock { assertFalse(resident.get()) }+ }assertFalse(operation.isCompleted)- assertFalse(cleanup.isCompleted)+ assertFalse(follower.isCompleted)assertTrue(resident.get())nativeReturn.countDown()operation.join()- cleanup.join()+ follower.join()assertFalse(resident.get())} finally {nativeReturn.countDown()@@ -136,16 +133,13 @@ class ModelMemoryPressureTest {fun failedWorkReleasesResourcesBeforeUnlocking() = runBlocking {val mutex = Mutex()var resident = false- var failed = false- try {+ val result = runCatching {withModelResources(mutex, { resident = false }) {resident = trueerror("Native operation failed")}- } catch (_: IllegalStateException) {- failed = true}- assertTrue(failed)+ assertTrue(result.exceptionOrNull() is IllegalStateException)assertFalse(resident)assertFalse(mutex.isLocked)}diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 178d24f80e..7cbc4e91ae 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -94,7 +94,6 @@ final class ChatViewModel: ObservableObject {private var warmupSuppressed = falseprivate var warmupTask: Task<Void, Never>?private var warmupAttempt: (owner: UUID, selection: ModelReadyKey)?- private var warmupObservation: AnyCancellable?private let provider: LlmProviderprivate let knowledgeEmbedding: KnowledgeEmbeddingConfig@@ -154,7 +153,6 @@ final class ChatViewModel: ObservableObject {provider.modelMaintenance = notesStorelet voiceTranscriber = VoiceTranscriptionService(provider: provider, assetStore: assetStore)- voiceTranscriber.modelMaintenance = notesStorelet knowledgeProvider = KnowledgeProvider(assetStore: assetStore)let knowledgeStore = KnowledgeStore(datasets: config.knowledgeDatasets, provider: knowledgeProvider)@@ -232,10 +230,6 @@ final class ChatViewModel: ObservableObject {}refreshDeviceCapability()refreshModelDownloadInfo()- warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)- .sink { [weak self] _ in- Task { @MainActor [weak self] in self?.refreshChatWarmup() }- }Task {async let knowledge: Void = knowledgeStore.bootstrap()async let notes: Void = notesStore.bootstrap()@@ -1278,6 +1272,7 @@ final class ChatViewModel: ObservableObject {return}if error is ModelMemoryUnavailable {+ guard !Task.isCancelled, activeGenerationId == generationId else { return }generationErrorMessage = error.localizedDescriptionreturn}@@ -1346,7 +1341,6 @@ final class ChatViewModel: ObservableObject {let normalSystemPrompt = systemPrompt()var activeCitations: [GroundedSource] = []var conversationPrepared = false- var generationFinalized = falselet buffer = OSAllocatedUnfairLock(initialState: StreamingBuffer())let uiUpdateInterval: TimeInterval = 0.05@@ -1396,8 +1390,6 @@ final class ChatViewModel: ObservableObject {) -> Void = {[self]result, preparation, citations in- guard !generationFinalized else { return }- generationFinalized = truelet snapshot = buffer.withLock { $0 }let interrupted: Boollet totalTimeMs: Int64?@@ -1431,9 +1423,8 @@ final class ChatViewModel: ObservableObject {preparation: preparation)}do {- let summary: GenerationSummaryif prompt.imageFiles.isEmpty {- summary = try await self.generatePreparedChat(+ try await self.generatePreparedChat(selection,request: ConversationRequest(sessionUuid: userNode.sessionId.uuidString,@@ -1473,8 +1464,8 @@ final class ChatViewModel: ObservableObject {hasOutput: { buffer.withLock { !$0.text.isEmpty } }, onToken: onToken)switch result {- case let .generated(generated):- summary = generated+ case let .generated(summary):+ finish(.success(summary), nil, activeCitations)case let .overflow(inputTokens, inputBudget, limits):overflowBypassMessageId = nilpendingOverflow = PendingOverflow(@@ -1493,9 +1484,8 @@ final class ChatViewModel: ObservableObject {return}}-- finish(.success(summary), nil, activeCitations)} catch {+ if activeGenerationId == generationId { refreshModelDownloadInfo() }finish(.failure(error), nil, activeCitations)}if embeddingAssetInvalid {@@ -1599,12 +1589,12 @@ final class ChatViewModel: ObservableObject {onProgress: @escaping @MainActor () -> Void,onPrepared: @escaping @MainActor (ConversationPreparation) -> Void,onFinished:- @escaping @MainActor (Result<GenerationSummary, Error>, ConversationPreparation?) ->+ @escaping @MainActor (Result<GenerationSummary, Error>, ConversationPreparation) ->Void,onToken: @escaping @Sendable (String) -> Void- ) async throws -> GenerationSummary {+ ) async throws {let db = chatDb- return try await provider.withConversationContext(+ try await provider.withConversationContext(selection,onProgress: { progress inTask { @MainActor in self.handleProgress(progress) }@@ -1637,10 +1627,8 @@ final class ChatViewModel: ObservableObject {onToken: onToken)onFinished(.success(summary), preparation)- return summary} catch {onFinished(.failure(error), preparation)- throw error}}}diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 464e24aa52..3748339cd9 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -82,7 +82,6 @@ final class VoiceTranscriptionService {typealias StateHandler = @MainActor @Sendable (VoiceInputState) -> Voidtypealias TranscriptHandler = @MainActor @Sendable (String) -> Void- weak var modelMaintenance: (any ModelMaintenance)?var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?private let provider: LlmProviderprivate let assetStore: AssetStore@@ -172,14 +171,8 @@ final class VoiceTranscriptionService {let pcm = recording.pcmlet owner = voiceOwner- let maintenance = modelMaintenance- let maintenanceScope = maintenance?.suspendMaintenance()transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in- defer {- maintenanceScope?.close()- Task.detached { await provider.releaseVoice(owner: owner) }- }- await maintenance?.awaitMaintenance()+ defer { Task.detached { await provider.releaseVoice(owner: owner) } }do {try await self?.downloadModelsIfNeeded(taskId: taskId,@@ -250,11 +243,7 @@ final class VoiceTranscriptionService {let taskId = beginVoiceTask()let downloadId = beginDownload(taskId: taskId)- let maintenance = modelMaintenance- let maintenanceScope = maintenance?.suspendMaintenance()transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in- defer { maintenanceScope?.close() }- await maintenance?.awaitMaintenance()do {try await self?.downloadModelsIfNeeded(taskId: taskId,diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex 86090e974f..a7afec916f 100644--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -195,7 +195,6 @@ actor LlmProvider {}}- private nonisolated let currentJobId = OSAllocatedUnfairLock<Int64?>(initialState: nil)private nonisolated let generationControl = OSAllocatedUnfairLock<ChatGenerationControl?>(initialState: nil)private let modelLoadGate = AsyncSerialGate()@@ -434,12 +433,10 @@ actor LlmProvider {try control.checkCancellation()generationControl.withLock { $0 = control }defer { generationControl.withLock { $0 = nil } }- let context = try model.newContext(- params: LlmContextParams(- contextSize: Int32(min(1024, loadedContext.contextSize())),- nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1)),- nBatch: 128- ))+ let context = try model.newTitleContext(+ chatContextSize: loadedContext.contextSize(),+ nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1))+ )let outputTokens = 48let inputTokens = max(0, min(2000, Int(context.contextSize()) - outputTokens))let titleMessages = try context.truncateTextChatMessages(@@ -516,7 +513,7 @@ actor LlmProvider {temperature: Float,maxTokens: Int?,contextOverride: LlmContext? = nil,- control: ChatGenerationControl? = nil,+ control: ChatGenerationControl,onToken: @escaping @Sendable (String) -> Void) async throws -> GenerationSummary {let capability = currentChatDeviceCapability()@@ -528,10 +525,8 @@ actor LlmProvider {domain: "LlmProvider", code: -1,userInfo: [NSLocalizedDescriptionKey: "Model not loaded"])}- currentJobId.withLock { $0 = nil }defer { context.releaseMultimodal() }- unloadTranscriptionModelIfLoaded()let asset = chatAsset(selection)let mmprojPath =imageFiles.isEmpty@@ -561,20 +556,15 @@ actor LlmProvider {grammar: nil)- let sink = CallbackSink { [currentJobId] event in- switch event {- case let .text(jobId, text, _):- currentJobId.withLock { $0 = jobId }- control?.setJob(jobId)+ let sink = CallbackSink { event in+ if case let .text(jobId, text, _) = event {+ control.setJob(jobId)onToken(text)- case .done:- currentJobId.withLock { $0 = nil }}}- defer { currentJobId.withLock { $0 = nil } }let summary = try await Task.detached {- try control?.checkCancellation()+ try control.checkCancellation()return try context.generateChatStream(request: request, callback: sink)}.value@@ -664,10 +654,6 @@ actor LlmProvider {nonisolated func stopGeneration(invalidatePreparation: Bool = false) {if let control = generationControl.withLock({ $0 }) {control.cancel(invalidatePreparation: invalidatePreparation)- return- }- if let jobId = currentJobId.withLock({ $0 }) {- llmCancel(jobId: jobId)} else {llmCancel(jobId: 0)}diff --git a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swiftindex 3e58d17d63..cba7099363 100644--- a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift+++ b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift@@ -2,17 +2,16 @@ import Foundationstruct ModelMemoryPressureState {static let cooldown: TimeInterval = 30- private var generation: UInt64 = 0- private var evictedGeneration: UInt64 = 0+ private var cleanupPending = falseprivate var retryAfter: TimeInterval = 0mutating func recordPressure(now: TimeInterval) {- generation &+= 1+ cleanupPending = trueretryAfter = now + Self.cooldown}func suppressesOptionalWork(now: TimeInterval) -> Bool {- generation != evictedGeneration || now < retryAfter+ cleanupPending || now < retryAfter}func requiresCleanup(appForeground: Bool, now: TimeInterval) -> Bool {@@ -20,7 +19,7 @@ struct ModelMemoryPressureState {}mutating func didCleanUp() {- evictedGeneration = generation+ cleanupPending = false}}diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swiftindex 65e6d05127..4b49c9d8e7 100644--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift@@ -18,11 +18,9 @@ private actor ModelOperationBarrier {}}-private final class RetainedPreparation: @unchecked Sendable {}-@MainActorfinal class ModelMemoryCoordinationTests: XCTestCase {- func testCooldownDoesNotAcknowledgePendingCleanup() {+ func testOptionalWorkWaitsForBothCleanupAndCooldown() {var pressure = ModelMemoryPressureState()pressure.recordPressure(now: 10)XCTAssertTrue(pressure.suppressesOptionalWork(now: 100))@@ -30,15 +28,12 @@ final class ModelMemoryCoordinationTests: XCTestCase {pressure.didCleanUp()XCTAssertFalse(pressure.suppressesOptionalWork(now: 100))XCTAssertFalse(pressure.requiresCleanup(appForeground: true, now: 100))- }- func testCooldownExpiresAfterCleanup() {- var pressure = ModelMemoryPressureState()- pressure.recordPressure(now: 10)+ pressure.recordPressure(now: 100)pressure.didCleanUp()- XCTAssertTrue(pressure.suppressesOptionalWork(now: 39.9))- XCTAssertFalse(pressure.suppressesOptionalWork(now: 40))- XCTAssertTrue(pressure.requiresCleanup(appForeground: false, now: 40))+ XCTAssertTrue(pressure.suppressesOptionalWork(now: 129.9))+ XCTAssertFalse(pressure.suppressesOptionalWork(now: 130))+ XCTAssertTrue(pressure.requiresCleanup(appForeground: false, now: 130))}func testNewPressureRestartsCooldownAndRequiresAnotherCleanup() {@@ -60,7 +55,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {let completed = expectation(description: "operations completed")completed.expectedFulfillmentCount = 2var events: [String] = []- let first = Task {+ Task {defer { completed.fulfill() }do {try await gate.withLock(@@ -74,7 +69,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {} catch { XCTFail("Unexpected failure: \(error)") }}await fulfillment(of: [cleaning], timeout: 2)- let second = Task {+ Task {defer { completed.fulfill() }queued.fulfill()do {@@ -85,8 +80,6 @@ final class ModelMemoryCoordinationTests: XCTestCase {XCTAssertEqual(events, ["enter", "work"])await barrier.release()await fulfillment(of: [completed], timeout: 2)- first.cancel()- second.cancel()XCTAssertEqual(events, ["enter", "work", "cleanup", "next"])}@@ -116,21 +109,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {if cancel { task.cancel() }await barrier.release()await fulfillment(of: [completed], timeout: 2)- task.cancel()XCTAssertEqual(events, ["work", "cleanup"])}}-- func testPreparationCancellationReferenceCanBeReleasedBeforeCleanup() {- let control = ChatGenerationControl()- weak var retained: RetainedPreparation?- do {- let preparation = RetainedPreparation()- retained = preparation- control.setPreparationCancellation { withExtendedLifetime(preparation) {} }- }- XCTAssertNotNil(retained)- control.clearPreparationCancellation()- XCTAssertNil(retained)- }}diff --git a/rust/apps/ensu/changes/mobile-model-memory.md b/rust/apps/ensu/changes/mobile-model-memory.mdindex 5e261e5365..5c472d9377 100644--- a/rust/apps/ensu/changes/mobile-model-memory.md+++ b/rust/apps/ensu/changes/mobile-model-memory.md@@ -1,2 +1 @@-- Keep chat models warm during knowledge and notes retrieval when memory allows, and limit memory used by voice, images, and chat titles on mobile.-- Warm the selected chat model when Chat opens, retain it across in-app navigation and brief attachment picking, and release idle resources on app backgrounding or memory pressure.+- Better model loading/unloading to warmup chat model and keep it in-memorydiff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rsindex a1d793c148..48d7fb4e4b 100644--- a/rust/bindings/uniffi/ensu/src/llm.rs+++ b/rust/bindings/uniffi/ensu/src/llm.rs@@ -110,6 +110,16 @@ impl LlmModel {Ok(Arc::new(LlmContext { handle }))}+ pub fn new_title_context(+ &self,+ chat_context_size: u32,+ n_threads: Option<i32>,+ ) -> Result<Arc<LlmContext>, LlmError> {+ let handle = llm::Context::new_title(&self.handle, chat_context_size, n_threads)+ .map_err(LlmError::from)?;+ Ok(Arc::new(LlmContext { handle }))+ }+pub fn new_embedding_context(&self,n_threads: Option<i32>,diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex c7b9374ca6..ce2b0f2501 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -3,11 +3,10 @@ mod knowledge;pub use knowledge::knowledge_dataset;pub use knowledge::{AttributionConfig, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig};pub(crate) use knowledge::{- EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL,- KNOWLEDGE_LICENSE_URL, KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE,- KNOWLEDGE_VECTORS_FILE, format_knowledge_document_prompt, is_path_safe_component,- knowledge_artifact_urls, knowledge_datasets, knowledge_embedding_config,- knowledge_index_contract,+ KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL, KNOWLEDGE_LICENSE_URL,+ KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,+ format_knowledge_document_prompt, is_path_safe_component, knowledge_artifact_urls,+ knowledge_datasets, knowledge_embedding_config, knowledge_index_contract,};#[derive(Debug, Clone)]diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rsindex b3d91eb520..ab6da1b1fb 100644--- a/rust/crates/ensu/src/config/knowledge.rs+++ b/rust/crates/ensu/src/config/knowledge.rs@@ -22,7 +22,7 @@ const EMBEDDING_MODEL_SHA256: &str =const EMBEDDING_TARGET_ID: &str = "embeddinggemma-300m-q8-0";const SOURCE_DIM: u32 = 768;const RETRIEVAL_DIM: u32 = 512;-pub(crate) const EMBEDDING_CONTEXT_SIZE: u32 = 2_048;+const EMBEDDING_CONTEXT_SIZE: u32 = 2_048;const MAX_HITS: u32 = 2;const MAX_CONTEXT_UTF8_BYTES: u32 = 6_000;const MODIFICATION_NOTICE: &str = "Adapted by Ente";diff --git a/rust/crates/ensu/src/llm/context.rs b/rust/crates/ensu/src/llm/context.rsindex e0cd62b1b7..468a33a941 100644--- a/rust/crates/ensu/src/llm/context.rs+++ b/rust/crates/ensu/src/llm/context.rs@@ -45,6 +45,17 @@ impl Context {Self::spawn(move || LocalContext::new(&model, params))}+ pub fn new_title(+ model: &ModelRef,+ chat_context_size: u32,+ n_threads: Option<i32>,+ ) -> Result<ContextRef, Error> {+ Self::new(+ model,+ super::memory::title_context_params(chat_context_size, n_threads),+ )+ }+pub fn new_embedding(model: &ModelRef,params: EmbeddingContextParams,diff --git a/rust/crates/ensu/src/llm/memory.rs b/rust/crates/ensu/src/llm/memory.rsindex 8e52ab64f1..8a2d06606c 100644--- a/rust/crates/ensu/src/llm/memory.rs+++ b/rust/crates/ensu/src/llm/memory.rs@@ -1,4 +1,5 @@-use crate::config::{EMBEDDING_CONTEXT_SIZE, ModelRuntimeSurface};+use super::ContextParams;+use crate::config::ModelRuntimeSurface;const MIB: u64 = 1024 * 1024;const MEMORY_RESERVE_BYTES: u64 = 256 * MIB;@@ -35,8 +36,7 @@ pub fn memory_budget(loaded_chat: Option<&ChatModelMemory>,query_bytes: Option<u64>,) -> MemoryBudget {- let token_bound = query_bytes- .map(|bytes| bytes.min(u64::from(EMBEDDING_CONTEXT_SIZE)) + EMBEDDING_QUERY_OVERHEAD);+ let token_bound = query_bytes.map(|bytes| bytes.saturating_add(EMBEDDING_QUERY_OVERHEAD));let short_query = token_bound.is_some_and(|tokens| tokens <= 512);if matches!(surface, ModelRuntimeSurface::Android)&& matches!(operation, MemoryOperation::Embedding)@@ -60,18 +60,36 @@ pub fn memory_budget(let additional_bytes = match operation {MemoryOperation::Embedding if short_query => 1024 * MIB,MemoryOperation::Embedding => 2560 * MIB,- MemoryOperation::Voice => 2048 * MIB,+ MemoryOperation::Voice => match surface {+ ModelRuntimeSurface::Android | ModelRuntimeSurface::Ios => 1280 * MIB,+ ModelRuntimeSurface::Desktop => 2048 * MIB,+ },MemoryOperation::Title => 512 * MIB,};MemoryBudget {required_bytes: additional_bytes + MEMORY_RESERVE_BYTES,- system_reserve_cap_bytes: None,+ system_reserve_cap_bytes: (matches!(surface, ModelRuntimeSurface::Android)+ && matches!(operation, MemoryOperation::Voice))+ .then_some(512 * MIB),+ }+}++pub(super) fn title_context_params(+ chat_context_size: u32,+ n_threads: Option<i32>,+) -> ContextParams {+ ContextParams {+ context_size: Some(chat_context_size.min(1024) as i32),+ n_threads,+ n_batch: Some(128),}}#[cfg(test)]mod tests {use super::*;+ use MemoryOperation::{Embedding, Title, Voice};+ use ModelRuntimeSurface::{Android, Desktop, Ios};#[test]fn required_work_preserves_minimum_reserve() {@@ -81,19 +99,29 @@ mod tests {}#[test]- fn generic_budgets_cover_query_boundaries_and_required_reserve() {- for (operation, query_bytes, required_mib) in [- (MemoryOperation::Embedding, None, 2816),- (MemoryOperation::Embedding, Some(0), 1280),- (MemoryOperation::Embedding, Some(448), 1280),- (MemoryOperation::Embedding, Some(449), 2816),- (MemoryOperation::Embedding, Some(u64::MAX), 2816),- (MemoryOperation::Voice, None, 2304),- (MemoryOperation::Title, None, 768),+ fn title_context_caps_memory_without_raising_the_chat_limit() {+ for (chat_context_size, expected) in [(512, 512), (4096, 1024)] {+ let params = title_context_params(chat_context_size, Some(2));+ assert_eq!(params.context_size, Some(expected));+ assert_eq!(params.n_batch, Some(128));+ }+ }++ #[test]+ fn generic_budgets_cover_query_boundaries_and_platform_reserves() {+ for (surface, operation, query_bytes, required_mib, reserve_cap) in [+ (Android, Embedding, None, 2816, None),+ (Android, Embedding, Some(448), 1280, None),+ (Android, Embedding, Some(449), 2816, None),+ (Android, Embedding, Some(u64::MAX), 2816, None),+ (Android, Title, None, 768, None),+ (Android, Voice, None, 1536, Some(512 * MIB)),+ (Ios, Voice, None, 1536, None),+ (Desktop, Voice, None, 2304, None),] {- let budget = memory_budget(ModelRuntimeSurface::Android, operation, None, query_bytes);+ let budget = memory_budget(surface, operation, None, query_bytes);assert_eq!(budget.required_bytes, required_mib * MIB);- assert_eq!(budget.system_reserve_cap_bytes, None);+ assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);}}@@ -105,69 +133,41 @@ mod tests {context_size: 12032,};for (query_bytes, required_mib, reserve_cap) in [- (Some(0), 1024, Some(512 * MIB)),(Some(192), 1024, Some(512 * MIB)),(Some(193), 1280, Some(512 * MIB)),(Some(448), 1280, Some(512 * MIB)),(Some(449), 2816, None),- (Some(u64::MAX), 2816, None),(None, 2816, None),] {- let budget = memory_budget(- ModelRuntimeSurface::Android,- MemoryOperation::Embedding,- Some(&chat),- query_bytes,- );+ let budget = memory_budget(Android, Embedding, Some(&chat), query_bytes);assert_eq!(budget.required_bytes, required_mib * MIB);assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);}- for (operation, required_mib) in [- (MemoryOperation::Voice, 2304),- (MemoryOperation::Title, 768),- ] {- let budget = memory_budget(- ModelRuntimeSurface::Android,- operation,- Some(&chat),- Some(0),- );+ for (operation, required_mib, reserve_cap) in+ [(Voice, 1536, Some(512 * MIB)), (Title, 768, None)]+ {+ let budget = memory_budget(Android, operation, Some(&chat), Some(0));assert_eq!(budget.required_bytes, required_mib * MIB);- assert_eq!(budget.system_reserve_cap_bytes, None);+ assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);}}#[test]fn unmeasured_platforms_models_and_contexts_keep_generic_budgets() {for (surface, model_id, model_bytes, context_size) in [- (ModelRuntimeSurface::Ios, "lfm-vl-1.6b", 695_752_480, 4096),- (- ModelRuntimeSurface::Desktop,- "lfm-vl-1.6b",- 695_752_480,- 4096,- ),- (ModelRuntimeSurface::Android, "gemma", 695_752_480, 4096),- (- ModelRuntimeSurface::Android,- "lfm-vl-1.6b",- 695_752_481,- 4096,- ),- (ModelRuntimeSurface::Android, "lfm-vl-1.6b", 695_752_480, 0),- (- ModelRuntimeSurface::Android,- "lfm-vl-1.6b",- 695_752_480,- 12033,- ),+ (Ios, "lfm-vl-1.6b", 695_752_480, 4096),+ (Desktop, "lfm-vl-1.6b", 695_752_480, 4096),+ (Android, "gemma", 695_752_480, 4096),+ (Android, "lfm-vl-1.6b", 695_752_481, 4096),+ (Android, "lfm-vl-1.6b", 695_752_480, 0),+ (Android, "lfm-vl-1.6b", 695_752_480, 12033),] {let chat = ChatModelMemory {model_id: model_id.into(),model_bytes,context_size,};- let budget = memory_budget(surface, MemoryOperation::Embedding, Some(&chat), Some(128));+ let budget = memory_budget(surface, Embedding, Some(&chat), Some(128));assert_eq!(budget.required_bytes, 1280 * MIB);assert_eq!(budget.system_reserve_cap_bytes, None);}