Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Improve mobile time to first token

setalp6c179eb87052
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 5646f47e57..74989294d6 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -58,7 +58,14 @@ class AppStore(
logRepository = logRepository,
)
private val modelSettingsActions =
- ModelSettingsActions(_state, sessionPreferences, llmProvider, logRepository)
+ ModelSettingsActions(
+ state = _state,
+ sessionPreferences = sessionPreferences,
+ llmProvider = llmProvider,
+ logRepository = logRepository,
+ notesStore = notesStore,
+ awaitKnowledgeReady = knowledgeStore::awaitEnabledPacksReady,
+ )
private val chatActions =
ChatStoreActions(
state = _state,
@@ -79,7 +86,7 @@ class AppStore(
fun bootstrap(scope: CoroutineScope) {
notesStore.bootstrap(scope)
chatActions.setScope(scope)
- modelSettingsActions.setScope(scope)
+ modelSettingsActions.bootstrap(scope)
refreshDeviceCapability(scope)
chatActions.bootstrap(scope)
modelSettingsActions.refreshModelDownloadInfo()
@@ -87,6 +94,10 @@ class AppStore(
_state.value = _state.value.copy(chat = _state.value.chat.copy(isModelStateKnown = true))
}
+ fun setChatActive(active: Boolean) = modelSettingsActions.setChatActive(active)
+
+ fun suppressChatWarmup() = modelSettingsActions.suppressChatWarmup()
+
fun refreshDeviceCapability(scope: CoroutineScope? = null) {
val capability = deviceCapabilityProvider.chatCapability()
val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemory
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index 8a15c355a9..c992dcf1f4 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -111,6 +111,7 @@ internal fun HomeNavigation(
popExitTransition = { backExit() },
) {
ChatView(
+ onVoiceInputStarted = store::suppressChatWarmup,
chatState = appState.chat,
assetStore = store.assetStore,
transcriber = store.transcriber,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
index 87658544ac..04cf4921c2 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
@@ -3,8 +3,10 @@
package io.ente.ensu
import android.annotation.SuppressLint
+import android.content.ComponentCallbacks2
import android.content.Context
import android.content.Intent
+import android.content.res.Configuration
import android.net.Uri
import android.provider.OpenableColumns
import android.webkit.MimeTypeMap
@@ -197,20 +199,47 @@ fun HomeView(
scope.launch { drawerState.close() }
}
- DisposableEffect(lifecycleOwner) {
+ DisposableEffect(lifecycleOwner, isChatRoute) {
val observer = LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_RESUME) {
latestStore.notesStore.setForeground(true)
latestStore.refreshModelDownloadInfo()
+ latestStore.setChatActive(isChatRoute)
} else if (event == Lifecycle.Event.ON_STOP) {
latestStore.notesStore.setForeground(false)
+ latestStore.setChatActive(false)
}
}
latestStore.notesStore.setForeground(
lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)
)
lifecycleOwner.lifecycle.addObserver(observer)
- onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
+ latestStore.setChatActive(
+ isChatRoute &&
+ lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)
+ )
+ onDispose {
+ latestStore.setChatActive(false)
+ lifecycleOwner.lifecycle.removeObserver(observer)
+ }
+ }
+
+ DisposableEffect(context) {
+ @Suppress("DEPRECATION", "OVERRIDE_DEPRECATION")
+ val callbacks =
+ object : ComponentCallbacks2 {
+ override fun onConfigurationChanged(config: Configuration) = Unit
+
+ override fun onLowMemory() = latestStore.suppressChatWarmup()
+
+ override fun onTrimMemory(level: Int) {
+ if (level >= ComponentCallbacks2.TRIM_MEMORY_RUNNING_LOW) {
+ latestStore.suppressChatWarmup()
+ }
+ }
+ }
+ context.applicationContext.registerComponentCallbacks(callbacks)
+ onDispose { context.applicationContext.unregisterComponentCallbacks(callbacks) }
}
LaunchedEffect(whatsNewService) {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 4ac3e26490..ebdfc391cb 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -1194,16 +1194,7 @@ internal class ChatStoreActions(
val buffer = StringBuilder()
try {
- llmProvider.ensureModelReady(selection) {}
- llmProvider.generateChat(
- selection = selection,
- messages = messages,
- imageFiles = emptyList(),
- temperature = 0.2f,
- maxTokens = 64,
- ) { token ->
- buffer.append(token)
- }
+ llmProvider.generateTitle(selection, messages) { token -> buffer.append(token) }
} catch (err: kotlinx.coroutines.CancellationException) {
throw err
} catch (err: Throwable) {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index b8a9e4b0d4..54da62eaf9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -73,6 +73,7 @@ fun ChatView(
onDismissUnsupportedDeviceDialog: () -> Unit,
onOverflowTrim: () -> Unit,
onOverflowCancel: () -> Unit,
+ onVoiceInputStarted: () -> Unit,
) {
val density = LocalDensity.current
val context = LocalContext.current
@@ -117,6 +118,7 @@ fun ChatView(
pendingVoiceSessionKey = null
if (granted) {
if (requestedSessionKey != null) {
+ onVoiceInputStarted()
voiceController.startRecording {
latestSessionKey == requestedSessionKey && latestCanStartVoiceInput
}
@@ -296,6 +298,7 @@ fun ChatView(
) == PackageManager.PERMISSION_GRANTED
if (canStartVoiceInput && hasMicrophonePermission) {
val requestedSessionKey = sessionKey
+ onVoiceInputStarted()
voiceController.startRecording {
latestSessionKey == requestedSessionKey &&
latestCanStartVoiceInput
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index aaf7823eaf..8bbbb099f2 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -26,6 +26,9 @@ import java.io.File
import java.util.concurrent.atomic.AtomicInteger
import kotlin.math.max
import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.isActive
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
@@ -54,6 +57,7 @@ class LlmProvider(
private val modelLoadMutex = Mutex()
private val activeDownloads = AtomicInteger()
private val embeddingAsset = knowledgeEmbeddingModelAsset()
+ private var chatWarmupOwner: String? = null
internal var modelMaintenance: ModelMaintenance? = null
private suspend fun <T> withModelContext(block: suspend () -> T): T =
@@ -86,6 +90,29 @@ class LlmProvider(
}
}
+ internal suspend fun prewarmChatModelIfDownloaded(
+ selection: LlmModelSelection,
+ owner: String,
+ ): Unit = withModelContext {
+ modelLoadMutex.withLock {
+ currentCoroutineContext().ensureActive()
+ if (loadedModel != null || !isChatModelReady(selection)) return@withLock
+ try {
+ ensureModelReadyLocked(selection, {}, allowRecovery = false, shouldDownload = false)
+ currentCoroutineContext().ensureActive()
+ chatWarmupOwner = owner
+ } catch (error: Throwable) {
+ unloadModel()
+ throw error
+ }
+ }
+ }
+
+ internal suspend fun releaseChatWarmup(owner: String) =
+ withContext(ioDispatcher) {
+ modelLoadMutex.withLock { if (chatWarmupOwner == owner) unloadModel() }
+ }
+
suspend fun ensureRequiredModelsReady(
selection: LlmModelSelection,
onProgress: (DownloadProgress) -> Unit,
@@ -161,6 +188,50 @@ class LlmProvider(
}
}
+ internal suspend fun generateTitle(
+ selection: LlmModelSelection,
+ messages: List<LlmMessage>,
+ onToken: (String) -> Unit,
+ ): Unit = withModelContext {
+ modelLoadMutex.withLock {
+ val coroutine = currentCoroutineContext()
+ coroutine.ensureActive()
+ ensureModelReadyLocked(selection, onProgress = {})
+ coroutine.ensureActive()
+ unloadTranscriptionModelIfLoaded()
+ val model = checkNotNull(loadedModel) { "Model not loaded" }
+ model
+ .newContext(
+ LlmContextParams(
+ contextSize = 2048,
+ nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),
+ nBatch = 128,
+ )
+ )
+ .use { context ->
+ coroutine.ensureActive()
+ val titleMessages =
+ context.truncateTextChatMessages(
+ messages.map { NativeChatMessage(it.roleString(), it.text) },
+ 2000u,
+ )
+ coroutine.ensureActive()
+ generateStreamWithCallback(
+ context,
+ chatRequest(
+ titleMessages,
+ emptyList(),
+ null,
+ 0.2f,
+ 48,
+ ),
+ onToken,
+ isCancelled = { !coroutine.isActive },
+ )
+ }
+ }
+ }
+
internal suspend fun <T> withConversationContext(
selection: LlmModelSelection,
block: suspend (LlmContext) -> T,
@@ -322,6 +393,7 @@ class LlmProvider(
}
private fun unloadModel() {
+ chatWarmupOwner = null
loadedContext?.destroy()
loadedContext = null
loadedModel?.destroy()
@@ -345,6 +417,7 @@ class LlmProvider(
allowRecovery: Boolean = true,
shouldDownload: Boolean = true,
) {
+ chatWarmupOwner = null
deviceCapabilityProvider.chatCapability().requireChatSupported()
val modelKey = LoadedModelKey(selection.id, selection.contextLength)
if (!backendInitialized) {
@@ -447,6 +520,7 @@ class LlmProvider(
context: LlmContext,
request: LlmChatRequest,
onToken: (String) -> Unit,
+ isCancelled: () -> Boolean = { false },
): NativeSummary {
val callback =
object : LlmGenerationEventCallback {
@@ -454,6 +528,7 @@ class LlmProvider(
when (event) {
is LlmGenerationEvent.Text -> {
currentJobId = event.jobId
+ if (isCancelled()) llmCancel(event.jobId)
if (event.text.isNotEmpty()) {
onToken(event.text)
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index 45d5bd8897..23293932d9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -11,12 +11,17 @@ import io.ente.ensu.coroutines.runCatchingCancellable
import io.ente.ensu.device.isChatSupported
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.logging.LogLevel
+import io.ente.ensu.notes.NotesStore
import io.ente.ensu.settings.IS_ENSU_PACKS_ENABLED
import io.ente.ensu.settings.SessionPreferencesDataStore
+import java.util.UUID
+import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
@@ -25,12 +30,24 @@ internal class ModelSettingsActions(
private val sessionPreferences: SessionPreferencesDataStore,
private val llmProvider: LlmProvider,
private val logRepository: FileLogRepository,
+ private val notesStore: NotesStore,
+ private val awaitKnowledgeReady: suspend () -> Unit,
) {
private var scope: CoroutineScope? = null
private var modelDownloadJob: Job? = null
+ private var chatActive = false
+ private var warmupSuppressed = false
+ private var warmupJob: Job? = null
+ private var warmupOwner: String? = null
+ private var warmupSelection: LlmModelSelection? = null
- fun setScope(scope: CoroutineScope) {
+ fun bootstrap(scope: CoroutineScope) {
this.scope = scope
+ scope.launch {
+ awaitKnowledgeReady()
+ notesStore.awaitReady()
+ combine(state, notesStore.state) { _, _ -> Unit }.collect { refreshChatWarmup() }
+ }
}
fun updateModelSettings(settings: ModelSettingsState) {
@@ -303,6 +320,79 @@ internal class ModelSettingsActions(
}
}
+ fun setChatActive(active: Boolean) {
+ if (chatActive == active) return
+ chatActive = active
+ if (active) warmupSuppressed = false
+ refreshChatWarmup()
+ }
+
+ fun suppressChatWarmup() {
+ warmupSuppressed = true
+ cancelChatWarmup()
+ }
+
+ private fun cancelChatWarmup() {
+ warmupJob?.cancel()
+ warmupJob = null
+ val owner = warmupOwner
+ warmupOwner = null
+ warmupSelection = null
+ if (owner != null) scope?.launch { llmProvider.releaseChatWarmup(owner) }
+ }
+
+ private fun refreshChatWarmup() {
+ val current = state.value
+ val selection = resolveSelection(current.modelSettings)
+ val eligible =
+ chatActive &&
+ !warmupSuppressed &&
+ current.chat.deviceCapability.isChatSupported() &&
+ current.knowledge.packs.values.none { it.enabled } &&
+ notesStore.state.value.collections.isEmpty()
+ if (!eligible || (warmupSelection != null && warmupSelection != selection)) {
+ cancelChatWarmup()
+ }
+ if (
+ !eligible ||
+ warmupOwner != null ||
+ current.chat.isGenerating ||
+ current.chat.isDownloading ||
+ !current.chat.isModelDownloaded
+ )
+ return
+ val scope = scope ?: return
+ val owner = UUID.randomUUID().toString()
+ warmupOwner = owner
+ warmupSelection = selection
+ warmupJob = scope.launch {
+ try {
+ delay(350)
+ awaitKnowledgeReady()
+ notesStore.awaitReady()
+ coroutineContext.ensureActive()
+ val latest = state.value
+ if (
+ latest.chat.isGenerating ||
+ latest.chat.isDownloading ||
+ latest.knowledge.packs.values.any { it.enabled } ||
+ notesStore.state.value.collections.isNotEmpty()
+ )
+ return@launch
+ llmProvider.prewarmChatModelIfDownloaded(selection, owner)
+ } catch (error: CancellationException) {
+ throw error
+ } catch (error: Exception) {
+ logRepository.log(
+ LogLevel.Info,
+ "Chat warm-up skipped",
+ tag = "Model",
+ throwable = error,
+ )
+ }
+ }
+ }
+
fun prewarmImageInferenceIfDownloaded() {
val scope = scope ?: return
val currentState = state.value
diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swift
index e9488c39af..51077d0e65 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift
@@ -144,6 +144,7 @@ struct ChatView: View {
}
.onChange(of: scenePhase) { newValue in
viewModel.notesStore.setForeground(newValue == .active)
+ viewModel.setChatActive(newValue == .active && !viewState.showSettings)
if newValue == .active {
viewModel.refreshModelDownloadInfo()
} else {
@@ -152,7 +153,17 @@ struct ChatView: View {
}
}
.environmentObject(viewModel.notesStore)
- .onAppear { viewModel.notesStore.setForeground(scenePhase == .active) }
+ .onAppear {
+ viewModel.notesStore.setForeground(scenePhase == .active)
+ viewModel.setChatActive(scenePhase == .active && !viewState.showSettings)
+ }
+ .onDisappear { viewModel.setChatActive(false) }
+ .onReceive(
+ NotificationCenter.default.publisher(
+ for: UIApplication.didReceiveMemoryWarningNotification)
+ ) { _ in
+ viewModel.suppressChatWarmup()
+ }
.sheet(isPresented: $viewState.showSettings) {
SettingsView(
knowledgeStore: viewModel.knowledgeStore,
@@ -164,6 +175,7 @@ struct ChatView: View {
)
}
.onChange(of: viewState.showSettings) { isPresented in
+ viewModel.setChatActive(!isPresented && scenePhase == .active)
if isPresented {
viewState.didDismissKeyboard = true
isInputFocused = false
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index f1efecae80..f79f06a3f9 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -1,3 +1,4 @@
+import Combine
import Foundation
import os
import SwiftUI
@@ -92,6 +93,11 @@ final class ChatViewModel: ObservableObject {
@Published var draftCursorMoveToken = UUID()
let knowledgeStore: KnowledgeStore
let notesStore: NotesStore
+ private var chatActive = false
+ private var warmupSuppressed = false
+ private var warmupTask: Task<Void, Never>?
+ private var warmupAttempt: (owner: UUID, selection: ModelReadyKey)?
+ private var warmupObservation: AnyCancellable?
private let provider: LlmProvider
private let knowledgeEmbedding: KnowledgeEmbeddingConfig
@@ -219,6 +225,10 @@ final class ChatViewModel: ObservableObject {
refreshDeviceCapability()
refreshModelDownloadInfo()
+ warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)
+ .sink { [weak self] _ in
+ Task { @MainActor [weak self] in self?.refreshChatWarmup() }
+ }
Task {
async let knowledge: Void = knowledgeStore.bootstrap()
async let notes: Void = notesStore.bootstrap()
@@ -226,6 +236,62 @@ final class ChatViewModel: ObservableObject {
}
}
+ func setChatActive(_ active: Bool) {
+ guard chatActive != active else { return }
+ chatActive = active
+ if active { warmupSuppressed = false }
+ refreshChatWarmup()
+ }
+
+ func suppressChatWarmup() {
+ warmupSuppressed = true
+ cancelChatWarmup()
+ }
+
+ private func cancelChatWarmup() {
+ warmupTask?.cancel()
+ warmupTask = nil
+ let owner = warmupAttempt?.owner
+ warmupAttempt = nil
+ if let owner {
+ Task { await provider.releaseChatWarmup(owner: owner) }
+ }
+ }
+
+ private func refreshChatWarmup() {
+ let selection = modelSettings.currentSelection()
+ let key = modelReadyKey(for: selection)
+ let eligible =
+ chatActive && !warmupSuppressed && !isChatUnsupported
+ && !voiceInputState.isWorking
+ && knowledgeStore.packs.allSatisfy { !$0.enabled } && notesStore.collections.isEmpty
+ guard eligible else {
+ cancelChatWarmup()
+ return
+ }
+ if let attempt = warmupAttempt, attempt.selection != key { cancelChatWarmup() }
+ guard warmupAttempt == nil, !isGenerating, !isDownloading,
+ isModelDownloaded
+ else { return }
+ let owner = UUID()
+ warmupAttempt = (owner, key)
+ warmupTask = Task { [weak self] in
+ guard let self else { return }
+ do {
+ try await Task.sleep(nanoseconds: 350_000_000)
+ await knowledgeStore.bootstrap()
+ await notesStore.bootstrap()
+ try Task.checkCancellation()
+ guard !isGenerating, !isDownloading, !voiceInputState.isWorking,
+ knowledgeStore.packs.allSatisfy({ !$0.enabled }), notesStore.collections.isEmpty
+ else { return }
+ try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)
+ } catch {
+ if !isCancellation(error) { logger.info("Chat warm-up skipped") }
+ }
+ }
+ }
+
var isChatUnsupported: Bool {
!deviceCapability.isChatSupported
}
@@ -400,6 +466,7 @@ final class ChatViewModel: ObservableObject {
}
func toggleVoiceInput() {
+ suppressChatWarmup()
if voiceInputState.isRecording {
voiceTranscriber.stopAndTranscribe(
onState: { [weak self] state in
@@ -743,6 +810,7 @@ final class ChatViewModel: ObservableObject {
}
func refreshModelDownloadInfo() {
+ defer { refreshChatWarmup() }
notesStore.modelReadinessChanged()
guard !isChatUnsupported else {
isDownloading = false
@@ -1997,7 +2065,7 @@ final class ChatViewModel: ObservableObject {
provider: provider,
selection: selection
)
- guard let summary else { return }
+ guard !Task.isCancelled, let summary else { return }
await MainActor.run {
self.applySessionSummary(sessionId: sessionId, summary: summary)
@@ -2098,13 +2166,9 @@ final class ChatViewModel: ObservableObject {
let buffer = OSAllocatedUnfairLock(initialState: "")
do {
- try await provider.ensureModelReady(selection) { _ in }
- _ = try await provider.generateChat(
+ try await provider.generateTitle(
selection,
- messages: messages,
- imageFiles: [],
- temperature: 0.2,
- maxTokens: 64
+ messages: messages
) { token in
buffer.withLock { $0.append(token) }
}
diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index c395155999..3a7006a5c1 100644
--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -136,6 +136,7 @@ actor LlmProvider {
private var currentModelKey: LoadedModelKey?
private var currentContextLength: Int?
private var backendInitialized = false
+ private var chatWarmupOwner: UUID?
private nonisolated let currentJobId = OSAllocatedUnfairLock<Int64?>(initialState: nil)
private nonisolated let generationControl = OSAllocatedUnfairLock<ChatGenerationControl?>(
initialState: nil)
@@ -245,12 +246,36 @@ actor LlmProvider {
}
}
+ func prewarmChatModelIfDownloaded(
+ _ selection: LlmModelSelection, owner: UUID
+ ) async throws {
+ try await withModelLock {
+ guard loadedModel == nil, isChatModelReady(selection) else { return }
+ do {
+ try await ensureModelReadyLocked(
+ selection, onProgress: { _ in }, allowRecovery: false, shouldDownload: false)
+ try Task.checkCancellation()
+ chatWarmupOwner = owner
+ } catch {
+ unloadModel()
+ throw error
+ }
+ }
+ }
+
+ func releaseChatWarmup(owner: UUID) async {
+ try? await modelLoadGate.withLock {
+ if chatWarmupOwner == owner { unloadModel() }
+ }
+ }
+
private func ensureModelReadyLocked(
_ selection: LlmModelSelection,
onProgress: @escaping @Sendable (DownloadProgress) -> Void,
allowRecovery: Bool,
shouldDownload: Bool = true
) async throws {
+ chatWarmupOwner = nil
let capability = currentChatDeviceCapability()
if !capability.isChatSupported {
throw UnsupportedDeviceMemoryError(capability: capability)
@@ -328,6 +353,46 @@ actor LlmProvider {
}
}
+ func generateTitle(
+ _ selection: LlmModelSelection,
+ messages: [LlmMessage],
+ onToken: @escaping @Sendable (String) -> Void
+ ) async throws {
+ let control = ChatGenerationControl()
+ try await withTaskCancellationHandler {
+ try await withModelLock {
+ try control.checkCancellation()
+ try await ensureModelReadyLocked(
+ selection, onProgress: { _ in }, allowRecovery: true)
+ try control.checkCancellation()
+ unloadTranscriptionModelIfLoaded()
+ guard let model = loadedModel else { throw CancellationError() }
+ generationControl.withLock { $0 = control }
+ defer { generationControl.withLock { $0 = nil } }
+ let context = try model.newContext(
+ params: LlmContextParams(
+ contextSize: 2048,
+ nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1)),
+ nBatch: 128
+ ))
+ let titleMessages = try context.truncateTextChatMessages(
+ messages: messages.map {
+ LlmChatMessage(role: $0.role.roleString, content: $0.text)
+ },
+ maxTokens: 2000
+ )
+ _ = try await generateChatLocked(
+ selection,
+ messages: titleMessages,
+ imageFiles: [], temperature: 0.2, maxTokens: 48,
+ contextOverride: context, control: control, onToken: onToken
+ )
+ }
+ } onCancel: {
+ control.cancel(invalidatePreparation: false)
+ }
+ }
+
func withConversationContext<T: Sendable>(
_ selection: LlmModelSelection,
_ operation: @MainActor (LlmContext, ChatGenerationControl) async throws -> T
@@ -378,6 +443,7 @@ actor LlmProvider {
imageFiles: [URL],
temperature: Float,
maxTokens: Int?,
+ contextOverride: LlmContext? = nil,
control: ChatGenerationControl? = nil,
onToken: @escaping @Sendable (String) -> Void
) async throws -> GenerationSummary {
@@ -385,7 +451,7 @@ actor LlmProvider {
if !capability.isChatSupported {
throw UnsupportedDeviceMemoryError(capability: capability)
}
- guard let context = loadedContext else {
+ guard let context = contextOverride ?? loadedContext else {
throw NSError(
domain: "LlmProvider", code: -1,
userInfo: [NSLocalizedDescriptionKey: "Model not loaded"])
@@ -568,6 +634,7 @@ actor LlmProvider {
}
private func unloadModel() {
+ chatWarmupOwner = nil
loadedContext = nil
loadedModel = nil
currentModelKey = nil
diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rs
index 041535f8a2..5620782587 100644
--- a/rust/bindings/uniffi/ensu/src/llm.rs
+++ b/rust/bindings/uniffi/ensu/src/llm.rs
@@ -135,6 +135,17 @@ impl LlmContext {
self.handle.embed(&text).map_err(LlmError::from)
}
+ pub fn truncate_text_chat_messages(
+ &self,
+ messages: Vec<LlmChatMessage>,
+ max_tokens: u32,
+ ) -> Result<Vec<LlmChatMessage>, LlmError> {
+ self.handle
+ .truncate_text_chat_messages(messages.into_iter().map(Into::into).collect(), max_tokens)
+ .map(|messages| messages.into_iter().map(Into::into).collect())
+ .map_err(LlmError::from)
+ }
+
pub fn generate_chat_stream(
&self,
request: LlmChatRequest,
diff --git a/rust/crates/ensu/src/llm/generate.rs b/rust/crates/ensu/src/llm/generate.rs
index f84fce3e16..0597997dc2 100644
--- a/rust/crates/ensu/src/llm/generate.rs
+++ b/rust/crates/ensu/src/llm/generate.rs
@@ -505,6 +505,59 @@ impl Context {
) -> Result<GenerationSummary, Error> {
generate_chat_stream(self, request, sink)
}
+
+ pub fn truncate_text_chat_messages(
+ &self,
+ messages: Vec<ChatMessage>,
+ max_tokens: u32,
+ ) -> Result<Vec<ChatMessage>, Error> {
+ let max_tokens = max_tokens.min(self.context_size());
+ let mut request = ChatRequest {
+ messages,
+ ..Default::default()
+ };
+ loop {
+ let tokens = self.measure_text_chat_prompt(&request)?;
+ if tokens <= max_tokens as usize {
+ return Ok(request.messages);
+ }
+ let index = request
+ .messages
+ .iter()
+ .rposition(|message| message.role != "system")
+ .ok_or(Error::PromptTooLong {
+ tokens,
+ context_size: max_tokens,
+ })?;
+ let content = std::mem::take(&mut request.messages[index].content);
+ let tokens = self.measure_text_chat_prompt(&request)?;
+ if tokens > max_tokens as usize {
+ if request.messages[..index]
+ .iter()
+ .all(|message| message.role == "system")
+ {
+ return Err(Error::PromptTooLong {
+ tokens,
+ context_size: max_tokens,
+ });
+ }
+ request.messages.remove(index);
+ continue;
+ }
+ let (mut low, mut high) = (0, content.len());
+ while low < high {
+ let end = content.ceil_char_boundary(low + (high - low).div_ceil(2));
+ request.messages[index].content = content[..end].to_owned();
+ if self.measure_text_chat_prompt(&request)? <= max_tokens as usize {
+ low = end;
+ } else {
+ high = content.floor_char_boundary(end - 1);
+ }
+ }
+ request.messages[index].content = content[..low].to_owned();
+ return Ok(request.messages);
+ }
+ }
}
fn generate_chat_stream(