π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Improve mobile time to first token
setalp6c179eb87052
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 5646f47e57..74989294d6 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -58,7 +58,14 @@ class AppStore(logRepository = logRepository,)private val modelSettingsActions =- ModelSettingsActions(_state, sessionPreferences, llmProvider, logRepository)+ ModelSettingsActions(+ state = _state,+ sessionPreferences = sessionPreferences,+ llmProvider = llmProvider,+ logRepository = logRepository,+ notesStore = notesStore,+ awaitKnowledgeReady = knowledgeStore::awaitEnabledPacksReady,+ )private val chatActions =ChatStoreActions(state = _state,@@ -79,7 +86,7 @@ class AppStore(fun bootstrap(scope: CoroutineScope) {notesStore.bootstrap(scope)chatActions.setScope(scope)- modelSettingsActions.setScope(scope)+ modelSettingsActions.bootstrap(scope)refreshDeviceCapability(scope)chatActions.bootstrap(scope)modelSettingsActions.refreshModelDownloadInfo()@@ -87,6 +94,10 @@ class AppStore(_state.value = _state.value.copy(chat = _state.value.chat.copy(isModelStateKnown = true))}+ fun setChatActive(active: Boolean) = modelSettingsActions.setChatActive(active)++ fun suppressChatWarmup() = modelSettingsActions.suppressChatWarmup()+fun refreshDeviceCapability(scope: CoroutineScope? = null) {val capability = deviceCapabilityProvider.chatCapability()val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemorydiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex 8a15c355a9..c992dcf1f4 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -111,6 +111,7 @@ internal fun HomeNavigation(popExitTransition = { backExit() },) {ChatView(+ onVoiceInputStarted = store::suppressChatWarmup,chatState = appState.chat,assetStore = store.assetStore,transcriber = store.transcriber,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.ktindex 87658544ac..04cf4921c2 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt@@ -3,8 +3,10 @@package io.ente.ensuimport android.annotation.SuppressLint+import android.content.ComponentCallbacks2import android.content.Contextimport android.content.Intent+import android.content.res.Configurationimport android.net.Uriimport android.provider.OpenableColumnsimport android.webkit.MimeTypeMap@@ -197,20 +199,47 @@ fun HomeView(scope.launch { drawerState.close() }}- DisposableEffect(lifecycleOwner) {+ DisposableEffect(lifecycleOwner, isChatRoute) {val observer = LifecycleEventObserver { _, event ->if (event == Lifecycle.Event.ON_RESUME) {latestStore.notesStore.setForeground(true)latestStore.refreshModelDownloadInfo()+ latestStore.setChatActive(isChatRoute)} else if (event == Lifecycle.Event.ON_STOP) {latestStore.notesStore.setForeground(false)+ latestStore.setChatActive(false)}}latestStore.notesStore.setForeground(lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED))lifecycleOwner.lifecycle.addObserver(observer)- onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }+ latestStore.setChatActive(+ isChatRoute &&+ lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)+ )+ onDispose {+ latestStore.setChatActive(false)+ lifecycleOwner.lifecycle.removeObserver(observer)+ }+ }++ DisposableEffect(context) {+ @Suppress("DEPRECATION", "OVERRIDE_DEPRECATION")+ val callbacks =+ object : ComponentCallbacks2 {+ override fun onConfigurationChanged(config: Configuration) = Unit++ override fun onLowMemory() = latestStore.suppressChatWarmup()++ override fun onTrimMemory(level: Int) {+ if (level >= ComponentCallbacks2.TRIM_MEMORY_RUNNING_LOW) {+ latestStore.suppressChatWarmup()+ }+ }+ }+ context.applicationContext.registerComponentCallbacks(callbacks)+ onDispose { context.applicationContext.unregisterComponentCallbacks(callbacks) }}LaunchedEffect(whatsNewService) {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex 4ac3e26490..ebdfc391cb 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -1194,16 +1194,7 @@ internal class ChatStoreActions(val buffer = StringBuilder()try {- llmProvider.ensureModelReady(selection) {}- llmProvider.generateChat(- selection = selection,- messages = messages,- imageFiles = emptyList(),- temperature = 0.2f,- maxTokens = 64,- ) { token ->- buffer.append(token)- }+ llmProvider.generateTitle(selection, messages) { token -> buffer.append(token) }} catch (err: kotlinx.coroutines.CancellationException) {throw err} catch (err: Throwable) {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex b8a9e4b0d4..54da62eaf9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -73,6 +73,7 @@ fun ChatView(onDismissUnsupportedDeviceDialog: () -> Unit,onOverflowTrim: () -> Unit,onOverflowCancel: () -> Unit,+ onVoiceInputStarted: () -> Unit,) {val density = LocalDensity.currentval context = LocalContext.current@@ -117,6 +118,7 @@ fun ChatView(pendingVoiceSessionKey = nullif (granted) {if (requestedSessionKey != null) {+ onVoiceInputStarted()voiceController.startRecording {latestSessionKey == requestedSessionKey && latestCanStartVoiceInput}@@ -296,6 +298,7 @@ fun ChatView() == PackageManager.PERMISSION_GRANTEDif (canStartVoiceInput && hasMicrophonePermission) {val requestedSessionKey = sessionKey+ onVoiceInputStarted()voiceController.startRecording {latestSessionKey == requestedSessionKey &&latestCanStartVoiceInputdiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex aaf7823eaf..8bbbb099f2 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -26,6 +26,9 @@ import java.io.Fileimport java.util.concurrent.atomic.AtomicIntegerimport kotlin.math.maximport kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.currentCoroutineContext+import kotlinx.coroutines.ensureActive+import kotlinx.coroutines.isActiveimport kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContext@@ -54,6 +57,7 @@ class LlmProvider(private val modelLoadMutex = Mutex()private val activeDownloads = AtomicInteger()private val embeddingAsset = knowledgeEmbeddingModelAsset()+ private var chatWarmupOwner: String? = nullinternal var modelMaintenance: ModelMaintenance? = nullprivate suspend fun <T> withModelContext(block: suspend () -> T): T =@@ -86,6 +90,29 @@ class LlmProvider(}}+ internal suspend fun prewarmChatModelIfDownloaded(+ selection: LlmModelSelection,+ owner: String,+ ): Unit = withModelContext {+ modelLoadMutex.withLock {+ currentCoroutineContext().ensureActive()+ if (loadedModel != null || !isChatModelReady(selection)) return@withLock+ try {+ ensureModelReadyLocked(selection, {}, allowRecovery = false, shouldDownload = false)+ currentCoroutineContext().ensureActive()+ chatWarmupOwner = owner+ } catch (error: Throwable) {+ unloadModel()+ throw error+ }+ }+ }++ internal suspend fun releaseChatWarmup(owner: String) =+ withContext(ioDispatcher) {+ modelLoadMutex.withLock { if (chatWarmupOwner == owner) unloadModel() }+ }+suspend fun ensureRequiredModelsReady(selection: LlmModelSelection,onProgress: (DownloadProgress) -> Unit,@@ -161,6 +188,50 @@ class LlmProvider(}}+ internal suspend fun generateTitle(+ selection: LlmModelSelection,+ messages: List<LlmMessage>,+ onToken: (String) -> Unit,+ ): Unit = withModelContext {+ modelLoadMutex.withLock {+ val coroutine = currentCoroutineContext()+ coroutine.ensureActive()+ ensureModelReadyLocked(selection, onProgress = {})+ coroutine.ensureActive()+ unloadTranscriptionModelIfLoaded()+ val model = checkNotNull(loadedModel) { "Model not loaded" }+ model+ .newContext(+ LlmContextParams(+ contextSize = 2048,+ nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),+ nBatch = 128,+ )+ )+ .use { context ->+ coroutine.ensureActive()+ val titleMessages =+ context.truncateTextChatMessages(+ messages.map { NativeChatMessage(it.roleString(), it.text) },+ 2000u,+ )+ coroutine.ensureActive()+ generateStreamWithCallback(+ context,+ chatRequest(+ titleMessages,+ emptyList(),+ null,+ 0.2f,+ 48,+ ),+ onToken,+ isCancelled = { !coroutine.isActive },+ )+ }+ }+ }+internal suspend fun <T> withConversationContext(selection: LlmModelSelection,block: suspend (LlmContext) -> T,@@ -322,6 +393,7 @@ class LlmProvider(}private fun unloadModel() {+ chatWarmupOwner = nullloadedContext?.destroy()loadedContext = nullloadedModel?.destroy()@@ -345,6 +417,7 @@ class LlmProvider(allowRecovery: Boolean = true,shouldDownload: Boolean = true,) {+ chatWarmupOwner = nulldeviceCapabilityProvider.chatCapability().requireChatSupported()val modelKey = LoadedModelKey(selection.id, selection.contextLength)if (!backendInitialized) {@@ -447,6 +520,7 @@ class LlmProvider(context: LlmContext,request: LlmChatRequest,onToken: (String) -> Unit,+ isCancelled: () -> Boolean = { false },): NativeSummary {val callback =object : LlmGenerationEventCallback {@@ -454,6 +528,7 @@ class LlmProvider(when (event) {is LlmGenerationEvent.Text -> {currentJobId = event.jobId+ if (isCancelled()) llmCancel(event.jobId)if (event.text.isNotEmpty()) {onToken(event.text)}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex 45d5bd8897..23293932d9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -11,12 +11,17 @@ import io.ente.ensu.coroutines.runCatchingCancellableimport io.ente.ensu.device.isChatSupportedimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.logging.LogLevel+import io.ente.ensu.notes.NotesStoreimport io.ente.ensu.settings.IS_ENSU_PACKS_ENABLEDimport io.ente.ensu.settings.SessionPreferencesDataStore+import java.util.UUID+import kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.CoroutineScopeimport kotlinx.coroutines.Jobimport kotlinx.coroutines.delay+import kotlinx.coroutines.ensureActiveimport kotlinx.coroutines.flow.MutableStateFlow+import kotlinx.coroutines.flow.combineimport kotlinx.coroutines.flow.updateimport kotlinx.coroutines.launch@@ -25,12 +30,24 @@ internal class ModelSettingsActions(private val sessionPreferences: SessionPreferencesDataStore,private val llmProvider: LlmProvider,private val logRepository: FileLogRepository,+ private val notesStore: NotesStore,+ private val awaitKnowledgeReady: suspend () -> Unit,) {private var scope: CoroutineScope? = nullprivate var modelDownloadJob: Job? = null+ private var chatActive = false+ private var warmupSuppressed = false+ private var warmupJob: Job? = null+ private var warmupOwner: String? = null+ private var warmupSelection: LlmModelSelection? = null- fun setScope(scope: CoroutineScope) {+ fun bootstrap(scope: CoroutineScope) {this.scope = scope+ scope.launch {+ awaitKnowledgeReady()+ notesStore.awaitReady()+ combine(state, notesStore.state) { _, _ -> Unit }.collect { refreshChatWarmup() }+ }}fun updateModelSettings(settings: ModelSettingsState) {@@ -303,6 +320,79 @@ internal class ModelSettingsActions(}}+ fun setChatActive(active: Boolean) {+ if (chatActive == active) return+ chatActive = active+ if (active) warmupSuppressed = false+ refreshChatWarmup()+ }++ fun suppressChatWarmup() {+ warmupSuppressed = true+ cancelChatWarmup()+ }++ private fun cancelChatWarmup() {+ warmupJob?.cancel()+ warmupJob = null+ val owner = warmupOwner+ warmupOwner = null+ warmupSelection = null+ if (owner != null) scope?.launch { llmProvider.releaseChatWarmup(owner) }+ }++ private fun refreshChatWarmup() {+ val current = state.value+ val selection = resolveSelection(current.modelSettings)+ val eligible =+ chatActive &&+ !warmupSuppressed &&+ current.chat.deviceCapability.isChatSupported() &&+ current.knowledge.packs.values.none { it.enabled } &&+ notesStore.state.value.collections.isEmpty()+ if (!eligible || (warmupSelection != null && warmupSelection != selection)) {+ cancelChatWarmup()+ }+ if (+ !eligible ||+ warmupOwner != null ||+ current.chat.isGenerating ||+ current.chat.isDownloading ||+ !current.chat.isModelDownloaded+ )+ return+ val scope = scope ?: return+ val owner = UUID.randomUUID().toString()+ warmupOwner = owner+ warmupSelection = selection+ warmupJob = scope.launch {+ try {+ delay(350)+ awaitKnowledgeReady()+ notesStore.awaitReady()+ coroutineContext.ensureActive()+ val latest = state.value+ if (+ latest.chat.isGenerating ||+ latest.chat.isDownloading ||+ latest.knowledge.packs.values.any { it.enabled } ||+ notesStore.state.value.collections.isNotEmpty()+ )+ return@launch+ llmProvider.prewarmChatModelIfDownloaded(selection, owner)+ } catch (error: CancellationException) {+ throw error+ } catch (error: Exception) {+ logRepository.log(+ LogLevel.Info,+ "Chat warm-up skipped",+ tag = "Model",+ throwable = error,+ )+ }+ }+ }+fun prewarmImageInferenceIfDownloaded() {val scope = scope ?: returnval currentState = state.valuediff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swiftindex e9488c39af..51077d0e65 100644--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift@@ -144,6 +144,7 @@ struct ChatView: View {}.onChange(of: scenePhase) { newValue inviewModel.notesStore.setForeground(newValue == .active)+ viewModel.setChatActive(newValue == .active && !viewState.showSettings)if newValue == .active {viewModel.refreshModelDownloadInfo()} else {@@ -152,7 +153,17 @@ struct ChatView: View {}}.environmentObject(viewModel.notesStore)- .onAppear { viewModel.notesStore.setForeground(scenePhase == .active) }+ .onAppear {+ viewModel.notesStore.setForeground(scenePhase == .active)+ viewModel.setChatActive(scenePhase == .active && !viewState.showSettings)+ }+ .onDisappear { viewModel.setChatActive(false) }+ .onReceive(+ NotificationCenter.default.publisher(+ for: UIApplication.didReceiveMemoryWarningNotification)+ ) { _ in+ viewModel.suppressChatWarmup()+ }.sheet(isPresented: $viewState.showSettings) {SettingsView(knowledgeStore: viewModel.knowledgeStore,@@ -164,6 +175,7 @@ struct ChatView: View {)}.onChange(of: viewState.showSettings) { isPresented in+ viewModel.setChatActive(!isPresented && scenePhase == .active)if isPresented {viewState.didDismissKeyboard = trueisInputFocused = falsediff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex f1efecae80..f79f06a3f9 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1,3 +1,4 @@+import Combineimport Foundationimport osimport SwiftUI@@ -92,6 +93,11 @@ final class ChatViewModel: ObservableObject {@Published var draftCursorMoveToken = UUID()let knowledgeStore: KnowledgeStorelet notesStore: NotesStore+ private var chatActive = false+ private var warmupSuppressed = false+ private var warmupTask: Task<Void, Never>?+ private var warmupAttempt: (owner: UUID, selection: ModelReadyKey)?+ private var warmupObservation: AnyCancellable?private let provider: LlmProviderprivate let knowledgeEmbedding: KnowledgeEmbeddingConfig@@ -219,6 +225,10 @@ final class ChatViewModel: ObservableObject {refreshDeviceCapability()refreshModelDownloadInfo()+ warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)+ .sink { [weak self] _ in+ Task { @MainActor [weak self] in self?.refreshChatWarmup() }+ }Task {async let knowledge: Void = knowledgeStore.bootstrap()async let notes: Void = notesStore.bootstrap()@@ -226,6 +236,62 @@ final class ChatViewModel: ObservableObject {}}+ func setChatActive(_ active: Bool) {+ guard chatActive != active else { return }+ chatActive = active+ if active { warmupSuppressed = false }+ refreshChatWarmup()+ }++ func suppressChatWarmup() {+ warmupSuppressed = true+ cancelChatWarmup()+ }++ private func cancelChatWarmup() {+ warmupTask?.cancel()+ warmupTask = nil+ let owner = warmupAttempt?.owner+ warmupAttempt = nil+ if let owner {+ Task { await provider.releaseChatWarmup(owner: owner) }+ }+ }++ private func refreshChatWarmup() {+ let selection = modelSettings.currentSelection()+ let key = modelReadyKey(for: selection)+ let eligible =+ chatActive && !warmupSuppressed && !isChatUnsupported+ && !voiceInputState.isWorking+ && knowledgeStore.packs.allSatisfy { !$0.enabled } && notesStore.collections.isEmpty+ guard eligible else {+ cancelChatWarmup()+ return+ }+ if let attempt = warmupAttempt, attempt.selection != key { cancelChatWarmup() }+ guard warmupAttempt == nil, !isGenerating, !isDownloading,+ isModelDownloaded+ else { return }+ let owner = UUID()+ warmupAttempt = (owner, key)+ warmupTask = Task { [weak self] in+ guard let self else { return }+ do {+ try await Task.sleep(nanoseconds: 350_000_000)+ await knowledgeStore.bootstrap()+ await notesStore.bootstrap()+ try Task.checkCancellation()+ guard !isGenerating, !isDownloading, !voiceInputState.isWorking,+ knowledgeStore.packs.allSatisfy({ !$0.enabled }), notesStore.collections.isEmpty+ else { return }+ try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)+ } catch {+ if !isCancellation(error) { logger.info("Chat warm-up skipped") }+ }+ }+ }+var isChatUnsupported: Bool {!deviceCapability.isChatSupported}@@ -400,6 +466,7 @@ final class ChatViewModel: ObservableObject {}func toggleVoiceInput() {+ suppressChatWarmup()if voiceInputState.isRecording {voiceTranscriber.stopAndTranscribe(onState: { [weak self] state in@@ -743,6 +810,7 @@ final class ChatViewModel: ObservableObject {}func refreshModelDownloadInfo() {+ defer { refreshChatWarmup() }notesStore.modelReadinessChanged()guard !isChatUnsupported else {isDownloading = false@@ -1997,7 +2065,7 @@ final class ChatViewModel: ObservableObject {provider: provider,selection: selection)- guard let summary else { return }+ guard !Task.isCancelled, let summary else { return }await MainActor.run {self.applySessionSummary(sessionId: sessionId, summary: summary)@@ -2098,13 +2166,9 @@ final class ChatViewModel: ObservableObject {let buffer = OSAllocatedUnfairLock(initialState: "")do {- try await provider.ensureModelReady(selection) { _ in }- _ = try await provider.generateChat(+ try await provider.generateTitle(selection,- messages: messages,- imageFiles: [],- temperature: 0.2,- maxTokens: 64+ messages: messages) { token inbuffer.withLock { $0.append(token) }}diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex c395155999..3a7006a5c1 100644--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -136,6 +136,7 @@ actor LlmProvider {private var currentModelKey: LoadedModelKey?private var currentContextLength: Int?private var backendInitialized = false+ private var chatWarmupOwner: UUID?private nonisolated let currentJobId = OSAllocatedUnfairLock<Int64?>(initialState: nil)private nonisolated let generationControl = OSAllocatedUnfairLock<ChatGenerationControl?>(initialState: nil)@@ -245,12 +246,36 @@ actor LlmProvider {}}+ func prewarmChatModelIfDownloaded(+ _ selection: LlmModelSelection, owner: UUID+ ) async throws {+ try await withModelLock {+ guard loadedModel == nil, isChatModelReady(selection) else { return }+ do {+ try await ensureModelReadyLocked(+ selection, onProgress: { _ in }, allowRecovery: false, shouldDownload: false)+ try Task.checkCancellation()+ chatWarmupOwner = owner+ } catch {+ unloadModel()+ throw error+ }+ }+ }++ func releaseChatWarmup(owner: UUID) async {+ try? await modelLoadGate.withLock {+ if chatWarmupOwner == owner { unloadModel() }+ }+ }+private func ensureModelReadyLocked(_ selection: LlmModelSelection,onProgress: @escaping @Sendable (DownloadProgress) -> Void,allowRecovery: Bool,shouldDownload: Bool = true) async throws {+ chatWarmupOwner = nillet capability = currentChatDeviceCapability()if !capability.isChatSupported {throw UnsupportedDeviceMemoryError(capability: capability)@@ -328,6 +353,46 @@ actor LlmProvider {}}+ func generateTitle(+ _ selection: LlmModelSelection,+ messages: [LlmMessage],+ onToken: @escaping @Sendable (String) -> Void+ ) async throws {+ let control = ChatGenerationControl()+ try await withTaskCancellationHandler {+ try await withModelLock {+ try control.checkCancellation()+ try await ensureModelReadyLocked(+ selection, onProgress: { _ in }, allowRecovery: true)+ try control.checkCancellation()+ unloadTranscriptionModelIfLoaded()+ guard let model = loadedModel else { throw CancellationError() }+ generationControl.withLock { $0 = control }+ defer { generationControl.withLock { $0 = nil } }+ let context = try model.newContext(+ params: LlmContextParams(+ contextSize: 2048,+ nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1)),+ nBatch: 128+ ))+ let titleMessages = try context.truncateTextChatMessages(+ messages: messages.map {+ LlmChatMessage(role: $0.role.roleString, content: $0.text)+ },+ maxTokens: 2000+ )+ _ = try await generateChatLocked(+ selection,+ messages: titleMessages,+ imageFiles: [], temperature: 0.2, maxTokens: 48,+ contextOverride: context, control: control, onToken: onToken+ )+ }+ } onCancel: {+ control.cancel(invalidatePreparation: false)+ }+ }+func withConversationContext<T: Sendable>(_ selection: LlmModelSelection,_ operation: @MainActor (LlmContext, ChatGenerationControl) async throws -> T@@ -378,6 +443,7 @@ actor LlmProvider {imageFiles: [URL],temperature: Float,maxTokens: Int?,+ contextOverride: LlmContext? = nil,control: ChatGenerationControl? = nil,onToken: @escaping @Sendable (String) -> Void) async throws -> GenerationSummary {@@ -385,7 +451,7 @@ actor LlmProvider {if !capability.isChatSupported {throw UnsupportedDeviceMemoryError(capability: capability)}- guard let context = loadedContext else {+ guard let context = contextOverride ?? loadedContext else {throw NSError(domain: "LlmProvider", code: -1,userInfo: [NSLocalizedDescriptionKey: "Model not loaded"])@@ -568,6 +634,7 @@ actor LlmProvider {}private func unloadModel() {+ chatWarmupOwner = nilloadedContext = nilloadedModel = nilcurrentModelKey = nildiff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rsindex 041535f8a2..5620782587 100644--- a/rust/bindings/uniffi/ensu/src/llm.rs+++ b/rust/bindings/uniffi/ensu/src/llm.rs@@ -135,6 +135,17 @@ impl LlmContext {self.handle.embed(&text).map_err(LlmError::from)}+ pub fn truncate_text_chat_messages(+ &self,+ messages: Vec<LlmChatMessage>,+ max_tokens: u32,+ ) -> Result<Vec<LlmChatMessage>, LlmError> {+ self.handle+ .truncate_text_chat_messages(messages.into_iter().map(Into::into).collect(), max_tokens)+ .map(|messages| messages.into_iter().map(Into::into).collect())+ .map_err(LlmError::from)+ }+pub fn generate_chat_stream(&self,request: LlmChatRequest,diff --git a/rust/crates/ensu/src/llm/generate.rs b/rust/crates/ensu/src/llm/generate.rsindex f84fce3e16..0597997dc2 100644--- a/rust/crates/ensu/src/llm/generate.rs+++ b/rust/crates/ensu/src/llm/generate.rs@@ -505,6 +505,59 @@ impl Context {) -> Result<GenerationSummary, Error> {generate_chat_stream(self, request, sink)}++ pub fn truncate_text_chat_messages(+ &self,+ messages: Vec<ChatMessage>,+ max_tokens: u32,+ ) -> Result<Vec<ChatMessage>, Error> {+ let max_tokens = max_tokens.min(self.context_size());+ let mut request = ChatRequest {+ messages,+ ..Default::default()+ };+ loop {+ let tokens = self.measure_text_chat_prompt(&request)?;+ if tokens <= max_tokens as usize {+ return Ok(request.messages);+ }+ let index = request+ .messages+ .iter()+ .rposition(|message| message.role != "system")+ .ok_or(Error::PromptTooLong {+ tokens,+ context_size: max_tokens,+ })?;+ let content = std::mem::take(&mut request.messages[index].content);+ let tokens = self.measure_text_chat_prompt(&request)?;+ if tokens > max_tokens as usize {+ if request.messages[..index]+ .iter()+ .all(|message| message.role == "system")+ {+ return Err(Error::PromptTooLong {+ tokens,+ context_size: max_tokens,+ });+ }+ request.messages.remove(index);+ continue;+ }+ let (mut low, mut high) = (0, content.len());+ while low < high {+ let end = content.ceil_char_boundary(low + (high - low).div_ceil(2));+ request.messages[index].content = content[..end].to_owned();+ if self.measure_text_chat_prompt(&request)? <= max_tokens as usize {+ low = end;+ } else {+ high = content.floor_char_boundary(end - 1);+ }+ }+ request.messages[index].content = content[..low].to_owned();+ return Ok(request.messages);+ }+ }}fn generate_chat_stream(