Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Refine mobile model memory coordination

setalp715916d7ebf6
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index e3d55bea77..7b57bd60eb 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -616,6 +616,7 @@ internal class ChatStoreActions(
} catch (error: kotlinx.coroutines.CancellationException) {
throw error
} catch (error: ModelMemoryUnavailable) {
+ if (!isActive() || stopRequested) return@launch
state.update {
it.copy(
chat =
@@ -761,20 +762,18 @@ internal class ChatStoreActions(
onModelProgress,
{ isActive() && state.value.modelSettings == settings },
)
- if (embeddingAssetInvalid) modelSettingsActions.refreshModelDownloadInfo()
- return@launch
+ } else {
+ generateImageConversation(
+ sessionId,
+ userMessage,
+ prompt,
+ selection,
+ modelSettingsActions.resolveTemperature(settings),
+ knowledgeHits,
+ onModelProgress,
+ { isActive() && state.value.modelSettings == settings },
+ )
}
-
- generateImageConversation(
- sessionId,
- userMessage,
- prompt,
- selection,
- modelSettingsActions.resolveTemperature(settings),
- knowledgeHits,
- onModelProgress,
- { isActive() && state.value.modelSettings == settings },
- )
if (embeddingAssetInvalid) modelSettingsActions.refreshModelDownloadInfo()
}
generationJob = activeJob
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index c0caaae14d..1aa2754e29 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -21,10 +21,7 @@ import io.ente.ensu.bindings.TranscriptionException
import io.ente.ensu.bindings.transcriptionModelAsset
import io.ente.ensu.bindings.voiceActivityModelAsset
import io.ente.ensu.llm.LlmProvider
-import io.ente.ensu.llm.ModelMaintenance
import io.ente.ensu.llm.ModelMemoryUnavailable
-import io.ente.ensu.llm.withMaintenanceSuspended
-import io.ente.ensu.notes.LocalNotesStore
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.util.UUID
@@ -90,14 +87,12 @@ internal fun rememberVoiceTranscriptionController(
onVoiceInputJob: (Job) -> Unit,
): VoiceTranscriptionController {
val lifecycleOwner = LocalLifecycleOwner.current
- val notes = LocalNotesStore.current
val controller =
- remember(llmProvider, notes) {
+ remember(llmProvider) {
VoiceTranscriptionController(
assetStore,
llmProvider,
onTranscript,
- notes,
onVoiceInputJob,
)
}
@@ -118,7 +113,6 @@ internal class VoiceTranscriptionController(
private val assetStore: AssetStore,
private val llmProvider: LlmProvider,
private val onTranscript: (String) -> Unit,
- private val maintenance: ModelMaintenance?,
private val onVoiceInputJob: (Job) -> Unit,
) {
private val modelAssets =
@@ -309,77 +303,65 @@ internal class VoiceTranscriptionController(
audioRecord = null
}
- private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) =
- maintenance.withMaintenanceSuspended {
- val owner = voiceOwner
- try {
- ensureTranscriptionModelDownloaded()
- transcriptionPreloadJob?.join()
- coroutineContext.ensureActive()
- val transcript =
- withContext(Dispatchers.IO) {
- withContext(Dispatchers.Main.immediate) {
- state = VoiceInputState.Transcribing
- }
- llmProvider.transcribe(
- sampleRate.toUInt(),
- pcm,
- )
- }
- .trim()
-
- if (transcript.isBlank()) {
- showTransientError("No speech detected.")
- } else {
- onTranscript(transcript)
- state = VoiceInputState.Idle
- }
- } catch (error: CancellationException) {
- throw error
- } catch (error: UnsatisfiedLinkError) {
- Log.w(TAG, "Voice transcription is not available on this device", error)
- state =
- VoiceInputState.Error("Voice transcription is not available on this device.")
- } catch (error: TranscriptionException) {
- Log.w(TAG, "Voice transcription failed: ${error.message}", error)
- state = VoiceInputState.Error(transcriptionErrorMessage(error))
- } catch (error: ModelMemoryUnavailable) {
- state = VoiceInputState.Error(checkNotNull(error.message))
- } catch (_: AssetDownloadException.Cancelled) {
- state = VoiceInputState.Idle
- } catch (error: AssetDownloadException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (_: LlmException.Cancelled) {
+ private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) {
+ val owner = voiceOwner
+ try {
+ ensureTranscriptionModelDownloaded()
+ transcriptionPreloadJob?.join()
+ coroutineContext.ensureActive()
+ state = VoiceInputState.Transcribing
+ val transcript = llmProvider.transcribe(sampleRate.toUInt(), pcm).trim()
+
+ if (transcript.isBlank()) {
+ showTransientError("No speech detected.")
+ } else {
+ onTranscript(transcript)
state = VoiceInputState.Idle
- } catch (error: LlmException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (error: Throwable) {
- Log.w(
- TAG,
- "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",
- error,
- )
- state = VoiceInputState.Error("Could not transcribe voice input.")
- } finally {
- withContext(NonCancellable) { llmProvider.releaseVoice(owner) }
}
+ } catch (error: CancellationException) {
+ throw error
+ } catch (error: UnsatisfiedLinkError) {
+ Log.w(TAG, "Voice transcription is not available on this device", error)
+ state = VoiceInputState.Error("Voice transcription is not available on this device.")
+ } catch (error: TranscriptionException) {
+ Log.w(TAG, "Voice transcription failed: ${error.message}", error)
+ state = VoiceInputState.Error(transcriptionErrorMessage(error))
+ } catch (error: ModelMemoryUnavailable) {
+ state = VoiceInputState.Error(checkNotNull(error.message))
+ } catch (_: AssetDownloadException.Cancelled) {
+ state = VoiceInputState.Idle
+ } catch (error: AssetDownloadException) {
+ Log.w(TAG, "Voice model download failed: ${error.message}", error)
+ state = VoiceInputState.Error(downloadErrorMessage)
+ } catch (_: LlmException.Cancelled) {
+ state = VoiceInputState.Idle
+ } catch (error: LlmException) {
+ Log.w(TAG, "Voice model download failed: ${error.message}", error)
+ state = VoiceInputState.Error(downloadErrorMessage)
+ } catch (error: Throwable) {
+ Log.w(
+ TAG,
+ "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",
+ error,
+ )
+ state = VoiceInputState.Error("Could not transcribe voice input.")
+ } finally {
+ withContext(NonCancellable) { llmProvider.releaseVoice(owner) }
}
+ }
private fun preloadTranscriptionModel() {
voiceOwner = UUID.randomUUID().toString()
val owner = voiceOwner
- transcriptionPreloadJob =
- launchVoiceWork(Dispatchers.IO) {
- try {
- llmProvider.prewarmVoice(owner)
- } catch (error: CancellationException) {
- throw error
- } catch (error: Throwable) {
- Log.d(TAG, "Voice preload skipped", error)
- }
+ transcriptionPreloadJob = launchVoiceWork {
+ try {
+ llmProvider.prewarmVoice(owner)
+ } catch (error: CancellationException) {
+ throw error
+ } catch (error: Throwable) {
+ Log.d(TAG, "Voice preload skipped", error)
}
+ }
}
private fun releaseVoice() {
@@ -387,7 +369,7 @@ internal class VoiceTranscriptionController(
transcriptionPreloadJob = null
val owner = voiceOwner
voiceOwner = UUID.randomUUID().toString()
- launchVoiceWork(NonCancellable + Dispatchers.IO) { llmProvider.releaseVoice(owner) }
+ launchVoiceWork(NonCancellable) { llmProvider.releaseVoice(owner) }
}
private suspend fun ensureTranscriptionModelDownloaded() {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt
index 3d35a39aa5..46924626a9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/AndroidDeviceCapabilityProvider.kt
@@ -20,18 +20,14 @@ class AndroidDeviceCapabilityProvider(context: Context) {
val manager = appContext.getSystemService(ActivityManager::class.java) ?: return null
val info = ActivityManager.MemoryInfo()
manager.getMemoryInfo(info)
- val process = android.os.Debug.MemoryInfo()
- android.os.Debug.getMemoryInfo(process)
- val processBytes = process.totalPss.toLong() * 1024
return when {
info.lowMemory -> 0uL
- info.totalMem <= 0 || info.availMem < 0 || process.totalPss <= 0 -> null
+ info.totalMem <= 0 || info.availMem < 0 -> null
else ->
modelMemoryHeadroomBytes(
info.totalMem,
info.availMem,
info.threshold,
- processBytes,
optionalWork,
optionalReserveCapBytes,
)
@@ -57,7 +53,6 @@ internal fun modelMemoryHeadroomBytes(
totalBytes: Long,
availableBytes: Long,
lowMemoryThreshold: Long,
- processBytes: Long,
optionalWork: Boolean = true,
optionalReserveCapBytes: Long? = null,
): ULong {
@@ -67,7 +62,5 @@ internal fun modelMemoryHeadroomBytes(
if (optionalWork) minOf(totalBytes / 10, optionalReserveCapBytes ?: Long.MAX_VALUE)
else 0,
)
- return minOf(availableBytes - systemReserve, totalBytes / 2 - processBytes)
- .coerceAtLeast(0)
- .toULong()
+ return (availableBytes - systemReserve).coerceAtLeast(0).toULong()
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index c26d656e58..e912d6000c 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -103,10 +103,10 @@ class LlmProvider(
return headroom != null && headroom >= budget.requiredBytes
}
- private fun hasRequiredWorkReserve(): Boolean {
- val headroom = deviceCapabilityProvider.modelMemoryHeadroom(optionalWork = false)
- return llmHasRequiredWorkReserve(headroom)
- }
+ private fun hasRequiredWorkReserve(): Boolean =
+ llmHasRequiredWorkReserve(
+ deviceCapabilityProvider.modelMemoryHeadroom(optionalWork = false)
+ )
private fun releaseProjector() {
loadedContext?.releaseMultimodal()
@@ -309,13 +309,7 @@ class LlmProvider(
): GenerationSummary {
deviceCapabilityProvider.chatCapability().requireChatSupported()
currentJobId = null
- unloadTranscriptionModelIfLoaded()
- val mmprojPath =
- if (imageFiles.isEmpty()) {
- null
- } else {
- assetStore.llmMmprojPath(chatAsset(selection))?.absolutePath
- }
+ val mmprojPath = assetStore.llmMmprojPath(chatAsset(selection))?.absolutePath
if (mmprojPath != null && !hasRequiredWorkReserve()) {
throw ModelMemoryUnavailable()
}
@@ -362,12 +356,9 @@ class LlmProvider(
coroutine.ensureActive()
val model = checkNotNull(loadedModel) { "Model not loaded" }
model
- .newContext(
- LlmContextParams(
- contextSize = minOf(1024, checkNotNull(currentContextLength)),
- nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),
- nBatch = 128,
- )
+ .newTitleContext(
+ chatContextSize = checkNotNull(currentContextLength).toUInt(),
+ nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),
)
.use { context ->
coroutine.ensureActive()
@@ -403,11 +394,7 @@ class LlmProvider(
ensureModelReadyLocked(selection, onProgress)
currentCoroutineContext().ensureActive()
unloadTranscriptionModelIfLoaded()
- try {
- block(requireNotNull(loadedContext))
- } finally {
- releaseProjector()
- }
+ block(requireNotNull(loadedContext))
}
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index bd974b29a4..0570f76e72 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -22,7 +22,6 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow
-import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
@@ -53,7 +52,7 @@ internal class ModelSettingsActions(
llmProvider.setAppForeground(appForeground)
awaitKnowledgeReady()
notesStore.awaitReady()
- combine(state, notesStore.state) { _, _ -> Unit }.collect { refreshChatWarmup() }
+ state.collect { refreshChatWarmup() }
}
}
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt
index 78d4e41035..7ecc628284 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ModelMemoryHeadroomTest.kt
@@ -5,42 +5,32 @@ import org.junit.Test
class ModelMemoryHeadroomTest {
@Test
- fun profileReserveRespectsPlatformAndProcessLimits() {
+ fun profileReserveRespectsPlatformThreshold() {
assertEquals(
1500uL,
- modelMemoryHeadroomBytes(12000, 2000, 200, 200, optionalReserveCapBytes = 500),
+ modelMemoryHeadroomBytes(12000, 2000, 200, optionalReserveCapBytes = 500),
)
assertEquals(
0uL,
- modelMemoryHeadroomBytes(12000, 400, 200, 200, optionalReserveCapBytes = 500),
+ modelMemoryHeadroomBytes(12000, 400, 200, optionalReserveCapBytes = 500),
)
assertEquals(
1200uL,
- modelMemoryHeadroomBytes(12000, 2000, 800, 200, optionalReserveCapBytes = 500),
+ modelMemoryHeadroomBytes(12000, 2000, 800, optionalReserveCapBytes = 500),
)
- assertEquals(
- 100uL,
- modelMemoryHeadroomBytes(12000, 2000, 200, 5900, optionalReserveCapBytes = 500),
- )
- }
-
- @Test
- fun systemPressureLimitsAnOtherwiseSmallProcess() {
- assertEquals(100uL, modelMemoryHeadroomBytes(4000, 500, 100, 100))
- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 300, 100, 100))
- assertEquals(100uL, modelMemoryHeadroomBytes(4000, 800, 700, 100))
}
@Test
- fun nativeProcessFootprintLimitsAnOtherwiseIdleSystem() {
- assertEquals(200uL, modelMemoryHeadroomBytes(4000, 3500, 100, 1800))
- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 3500, 100, 2100))
+ fun optionalHeadroomPreservesSystemReserve() {
+ assertEquals(100uL, modelMemoryHeadroomBytes(4000, 500, 100))
+ assertEquals(0uL, modelMemoryHeadroomBytes(4000, 300, 100))
+ assertEquals(100uL, modelMemoryHeadroomBytes(4000, 800, 700))
+ assertEquals(3100uL, modelMemoryHeadroomBytes(4000, 3500, 100))
}
@Test
- fun requiredWorkPreservesPlatformAndProcessLimitsWithoutTheSpeculativeReserve() {
- assertEquals(400uL, modelMemoryHeadroomBytes(4000, 500, 100, 100, optionalWork = false))
- assertEquals(0uL, modelMemoryHeadroomBytes(4000, 90, 100, 100, optionalWork = false))
- assertEquals(200uL, modelMemoryHeadroomBytes(4000, 3500, 100, 1800, optionalWork = false))
+ fun requiredWorkPreservesPlatformThresholdWithoutTheSpeculativeReserve() {
+ assertEquals(400uL, modelMemoryHeadroomBytes(4000, 500, 100, optionalWork = false))
+ assertEquals(0uL, modelMemoryHeadroomBytes(4000, 90, 100, optionalWork = false))
}
}
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
index 1f581f545a..82db127727 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/llm/ModelMemoryPressureTest.kt
@@ -19,7 +19,7 @@ import org.junit.Test
class ModelMemoryPressureTest {
@Test
- fun evictionSurvivesTheAdmissionCooldownUntilResourcesAreReleased() {
+ fun optionalWorkWaitsForBothReleaseAndCooldown() {
var now = 0L
val pressure = ModelMemoryPressure { now }
pressure.requestEviction()
@@ -32,18 +32,12 @@ class ModelMemoryPressureTest {
pressure.didRelease(eviction)
assertNull(pressure.pendingEviction())
assertFalse(pressure.suppressesOptionalWork())
- }
-
- @Test
- fun releaseDoesNotBypassCooldown() {
- var now = 0L
- val pressure = ModelMemoryPressure { now }
pressure.requestEviction()
pressure.didRelease(pressure.pendingEviction()!!)
- now = 29_999
+ now = 89_999
assertTrue(pressure.suppressesOptionalWork())
- now = 30_000
+ now = 90_000
assertFalse(pressure.suppressesOptionalWork())
}
@@ -87,10 +81,11 @@ class ModelMemoryPressureTest {
}
}
releaseRequested = true
- val cleanup = launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { release() } }
+ val follower =
+ launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { assertFalse(resident) } }
mutex.unlock()
operation.join()
- cleanup.join()
+ follower.join()
assertEquals(listOf("release", "allocate", "release"), events)
assertFalse(resident)
@@ -117,14 +112,16 @@ class ModelMemoryPressureTest {
entered.await()
releaseRequested.set(true)
operation.cancel()
- val cleanup =
- launch(start = CoroutineStart.UNDISPATCHED) { mutex.withLock { release() } }
+ val follower =
+ launch(start = CoroutineStart.UNDISPATCHED) {
+ mutex.withLock { assertFalse(resident.get()) }
+ }
assertFalse(operation.isCompleted)
- assertFalse(cleanup.isCompleted)
+ assertFalse(follower.isCompleted)
assertTrue(resident.get())
nativeReturn.countDown()
operation.join()
- cleanup.join()
+ follower.join()
assertFalse(resident.get())
} finally {
nativeReturn.countDown()
@@ -136,16 +133,13 @@ class ModelMemoryPressureTest {
fun failedWorkReleasesResourcesBeforeUnlocking() = runBlocking {
val mutex = Mutex()
var resident = false
- var failed = false
- try {
+ val result = runCatching {
withModelResources(mutex, { resident = false }) {
resident = true
error("Native operation failed")
}
- } catch (_: IllegalStateException) {
- failed = true
}
- assertTrue(failed)
+ assertTrue(result.exceptionOrNull() is IllegalStateException)
assertFalse(resident)
assertFalse(mutex.isLocked)
}
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 178d24f80e..7cbc4e91ae 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -94,7 +94,6 @@ final class ChatViewModel: ObservableObject {
private var warmupSuppressed = false
private var warmupTask: Task<Void, Never>?
private var warmupAttempt: (owner: UUID, selection: ModelReadyKey)?
- private var warmupObservation: AnyCancellable?
private let provider: LlmProvider
private let knowledgeEmbedding: KnowledgeEmbeddingConfig
@@ -154,7 +153,6 @@ final class ChatViewModel: ObservableObject {
provider.modelMaintenance = notesStore
let voiceTranscriber = VoiceTranscriptionService(
provider: provider, assetStore: assetStore)
- voiceTranscriber.modelMaintenance = notesStore
let knowledgeProvider = KnowledgeProvider(assetStore: assetStore)
let knowledgeStore = KnowledgeStore(
datasets: config.knowledgeDatasets, provider: knowledgeProvider)
@@ -232,10 +230,6 @@ final class ChatViewModel: ObservableObject {
}
refreshDeviceCapability()
refreshModelDownloadInfo()
- warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)
- .sink { [weak self] _ in
- Task { @MainActor [weak self] in self?.refreshChatWarmup() }
- }
Task {
async let knowledge: Void = knowledgeStore.bootstrap()
async let notes: Void = notesStore.bootstrap()
@@ -1278,6 +1272,7 @@ final class ChatViewModel: ObservableObject {
return
}
if error is ModelMemoryUnavailable {
+ guard !Task.isCancelled, activeGenerationId == generationId else { return }
generationErrorMessage = error.localizedDescription
return
}
@@ -1346,7 +1341,6 @@ final class ChatViewModel: ObservableObject {
let normalSystemPrompt = systemPrompt()
var activeCitations: [GroundedSource] = []
var conversationPrepared = false
- var generationFinalized = false
let buffer = OSAllocatedUnfairLock(initialState: StreamingBuffer())
let uiUpdateInterval: TimeInterval = 0.05
@@ -1396,8 +1390,6 @@ final class ChatViewModel: ObservableObject {
) -> Void = {
[self]
result, preparation, citations in
- guard !generationFinalized else { return }
- generationFinalized = true
let snapshot = buffer.withLock { $0 }
let interrupted: Bool
let totalTimeMs: Int64?
@@ -1431,9 +1423,8 @@ final class ChatViewModel: ObservableObject {
preparation: preparation)
}
do {
- let summary: GenerationSummary
if prompt.imageFiles.isEmpty {
- summary = try await self.generatePreparedChat(
+ try await self.generatePreparedChat(
selection,
request: ConversationRequest(
sessionUuid: userNode.sessionId.uuidString,
@@ -1473,8 +1464,8 @@ final class ChatViewModel: ObservableObject {
hasOutput: { buffer.withLock { !$0.text.isEmpty } }, onToken: onToken
)
switch result {
- case let .generated(generated):
- summary = generated
+ case let .generated(summary):
+ finish(.success(summary), nil, activeCitations)
case let .overflow(inputTokens, inputBudget, limits):
overflowBypassMessageId = nil
pendingOverflow = PendingOverflow(
@@ -1493,9 +1484,8 @@ final class ChatViewModel: ObservableObject {
return
}
}
-
- finish(.success(summary), nil, activeCitations)
} catch {
+ if activeGenerationId == generationId { refreshModelDownloadInfo() }
finish(.failure(error), nil, activeCitations)
}
if embeddingAssetInvalid {
@@ -1599,12 +1589,12 @@ final class ChatViewModel: ObservableObject {
onProgress: @escaping @MainActor () -> Void,
onPrepared: @escaping @MainActor (ConversationPreparation) -> Void,
onFinished:
- @escaping @MainActor (Result<GenerationSummary, Error>, ConversationPreparation?) ->
+ @escaping @MainActor (Result<GenerationSummary, Error>, ConversationPreparation) ->
Void,
onToken: @escaping @Sendable (String) -> Void
- ) async throws -> GenerationSummary {
+ ) async throws {
let db = chatDb
- return try await provider.withConversationContext(
+ try await provider.withConversationContext(
selection,
onProgress: { progress in
Task { @MainActor in self.handleProgress(progress) }
@@ -1637,10 +1627,8 @@ final class ChatViewModel: ObservableObject {
onToken: onToken
)
onFinished(.success(summary), preparation)
- return summary
} catch {
onFinished(.failure(error), preparation)
- throw error
}
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 464e24aa52..3748339cd9 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -82,7 +82,6 @@ final class VoiceTranscriptionService {
typealias StateHandler = @MainActor @Sendable (VoiceInputState) -> Void
typealias TranscriptHandler = @MainActor @Sendable (String) -> Void
- weak var modelMaintenance: (any ModelMaintenance)?
var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?
private let provider: LlmProvider
private let assetStore: AssetStore
@@ -172,14 +171,8 @@ final class VoiceTranscriptionService {
let pcm = recording.pcm
let owner = voiceOwner
- let maintenance = modelMaintenance
- let maintenanceScope = maintenance?.suspendMaintenance()
transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
- defer {
- maintenanceScope?.close()
- Task.detached { await provider.releaseVoice(owner: owner) }
- }
- await maintenance?.awaitMaintenance()
+ defer { Task.detached { await provider.releaseVoice(owner: owner) } }
do {
try await self?.downloadModelsIfNeeded(
taskId: taskId,
@@ -250,11 +243,7 @@ final class VoiceTranscriptionService {
let taskId = beginVoiceTask()
let downloadId = beginDownload(taskId: taskId)
- let maintenance = modelMaintenance
- let maintenanceScope = maintenance?.suspendMaintenance()
transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
- defer { maintenanceScope?.close() }
- await maintenance?.awaitMaintenance()
do {
try await self?.downloadModelsIfNeeded(
taskId: taskId,
diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index 86090e974f..a7afec916f 100644
--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -195,7 +195,6 @@ actor LlmProvider {
}
}
- private nonisolated let currentJobId = OSAllocatedUnfairLock<Int64?>(initialState: nil)
private nonisolated let generationControl = OSAllocatedUnfairLock<ChatGenerationControl?>(
initialState: nil)
private let modelLoadGate = AsyncSerialGate()
@@ -434,12 +433,10 @@ actor LlmProvider {
try control.checkCancellation()
generationControl.withLock { $0 = control }
defer { generationControl.withLock { $0 = nil } }
- let context = try model.newContext(
- params: LlmContextParams(
- contextSize: Int32(min(1024, loadedContext.contextSize())),
- nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1)),
- nBatch: 128
- ))
+ let context = try model.newTitleContext(
+ chatContextSize: loadedContext.contextSize(),
+ nThreads: Int32(max(1, ProcessInfo.processInfo.activeProcessorCount - 1))
+ )
let outputTokens = 48
let inputTokens = max(0, min(2000, Int(context.contextSize()) - outputTokens))
let titleMessages = try context.truncateTextChatMessages(
@@ -516,7 +513,7 @@ actor LlmProvider {
temperature: Float,
maxTokens: Int?,
contextOverride: LlmContext? = nil,
- control: ChatGenerationControl? = nil,
+ control: ChatGenerationControl,
onToken: @escaping @Sendable (String) -> Void
) async throws -> GenerationSummary {
let capability = currentChatDeviceCapability()
@@ -528,10 +525,8 @@ actor LlmProvider {
domain: "LlmProvider", code: -1,
userInfo: [NSLocalizedDescriptionKey: "Model not loaded"])
}
- currentJobId.withLock { $0 = nil }
defer { context.releaseMultimodal() }
- unloadTranscriptionModelIfLoaded()
let asset = chatAsset(selection)
let mmprojPath =
imageFiles.isEmpty
@@ -561,20 +556,15 @@ actor LlmProvider {
grammar: nil
)
- let sink = CallbackSink { [currentJobId] event in
- switch event {
- case let .text(jobId, text, _):
- currentJobId.withLock { $0 = jobId }
- control?.setJob(jobId)
+ let sink = CallbackSink { event in
+ if case let .text(jobId, text, _) = event {
+ control.setJob(jobId)
onToken(text)
- case .done:
- currentJobId.withLock { $0 = nil }
}
}
- defer { currentJobId.withLock { $0 = nil } }
let summary = try await Task.detached {
- try control?.checkCancellation()
+ try control.checkCancellation()
return try context.generateChatStream(request: request, callback: sink)
}.value
@@ -664,10 +654,6 @@ actor LlmProvider {
nonisolated func stopGeneration(invalidatePreparation: Bool = false) {
if let control = generationControl.withLock({ $0 }) {
control.cancel(invalidatePreparation: invalidatePreparation)
- return
- }
- if let jobId = currentJobId.withLock({ $0 }) {
- llmCancel(jobId: jobId)
} else {
llmCancel(jobId: 0)
}
diff --git a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
index 3e58d17d63..cba7099363 100644
--- a/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
+++ b/apple/apps/ensu/Ensu/Services/ModelMemoryCoordination.swift
@@ -2,17 +2,16 @@ import Foundation
struct ModelMemoryPressureState {
static let cooldown: TimeInterval = 30
- private var generation: UInt64 = 0
- private var evictedGeneration: UInt64 = 0
+ private var cleanupPending = false
private var retryAfter: TimeInterval = 0
mutating func recordPressure(now: TimeInterval) {
- generation &+= 1
+ cleanupPending = true
retryAfter = now + Self.cooldown
}
func suppressesOptionalWork(now: TimeInterval) -> Bool {
- generation != evictedGeneration || now < retryAfter
+ cleanupPending || now < retryAfter
}
func requiresCleanup(appForeground: Bool, now: TimeInterval) -> Bool {
@@ -20,7 +19,7 @@ struct ModelMemoryPressureState {
}
mutating func didCleanUp() {
- evictedGeneration = generation
+ cleanupPending = false
}
}
diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
index 65e6d05127..4b49c9d8e7 100644
--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
@@ -18,11 +18,9 @@ private actor ModelOperationBarrier {
}
}
-private final class RetainedPreparation: @unchecked Sendable {}
-
@MainActor
final class ModelMemoryCoordinationTests: XCTestCase {
- func testCooldownDoesNotAcknowledgePendingCleanup() {
+ func testOptionalWorkWaitsForBothCleanupAndCooldown() {
var pressure = ModelMemoryPressureState()
pressure.recordPressure(now: 10)
XCTAssertTrue(pressure.suppressesOptionalWork(now: 100))
@@ -30,15 +28,12 @@ final class ModelMemoryCoordinationTests: XCTestCase {
pressure.didCleanUp()
XCTAssertFalse(pressure.suppressesOptionalWork(now: 100))
XCTAssertFalse(pressure.requiresCleanup(appForeground: true, now: 100))
- }
- func testCooldownExpiresAfterCleanup() {
- var pressure = ModelMemoryPressureState()
- pressure.recordPressure(now: 10)
+ pressure.recordPressure(now: 100)
pressure.didCleanUp()
- XCTAssertTrue(pressure.suppressesOptionalWork(now: 39.9))
- XCTAssertFalse(pressure.suppressesOptionalWork(now: 40))
- XCTAssertTrue(pressure.requiresCleanup(appForeground: false, now: 40))
+ XCTAssertTrue(pressure.suppressesOptionalWork(now: 129.9))
+ XCTAssertFalse(pressure.suppressesOptionalWork(now: 130))
+ XCTAssertTrue(pressure.requiresCleanup(appForeground: false, now: 130))
}
func testNewPressureRestartsCooldownAndRequiresAnotherCleanup() {
@@ -60,7 +55,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {
let completed = expectation(description: "operations completed")
completed.expectedFulfillmentCount = 2
var events: [String] = []
- let first = Task {
+ Task {
defer { completed.fulfill() }
do {
try await gate.withLock(
@@ -74,7 +69,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {
} catch { XCTFail("Unexpected failure: \(error)") }
}
await fulfillment(of: [cleaning], timeout: 2)
- let second = Task {
+ Task {
defer { completed.fulfill() }
queued.fulfill()
do {
@@ -85,8 +80,6 @@ final class ModelMemoryCoordinationTests: XCTestCase {
XCTAssertEqual(events, ["enter", "work"])
await barrier.release()
await fulfillment(of: [completed], timeout: 2)
- first.cancel()
- second.cancel()
XCTAssertEqual(events, ["enter", "work", "cleanup", "next"])
}
@@ -116,21 +109,7 @@ final class ModelMemoryCoordinationTests: XCTestCase {
if cancel { task.cancel() }
await barrier.release()
await fulfillment(of: [completed], timeout: 2)
- task.cancel()
XCTAssertEqual(events, ["work", "cleanup"])
}
}
-
- func testPreparationCancellationReferenceCanBeReleasedBeforeCleanup() {
- let control = ChatGenerationControl()
- weak var retained: RetainedPreparation?
- do {
- let preparation = RetainedPreparation()
- retained = preparation
- control.setPreparationCancellation { withExtendedLifetime(preparation) {} }
- }
- XCTAssertNotNil(retained)
- control.clearPreparationCancellation()
- XCTAssertNil(retained)
- }
}
diff --git a/rust/apps/ensu/changes/mobile-model-memory.md b/rust/apps/ensu/changes/mobile-model-memory.md
index 5e261e5365..5c472d9377 100644
--- a/rust/apps/ensu/changes/mobile-model-memory.md
+++ b/rust/apps/ensu/changes/mobile-model-memory.md
@@ -1,2 +1 @@
-- Keep chat models warm during knowledge and notes retrieval when memory allows, and limit memory used by voice, images, and chat titles on mobile.
-- Warm the selected chat model when Chat opens, retain it across in-app navigation and brief attachment picking, and release idle resources on app backgrounding or memory pressure.
+- Better model loading/unloading to warmup chat model and keep it in-memory
diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rs
index a1d793c148..48d7fb4e4b 100644
--- a/rust/bindings/uniffi/ensu/src/llm.rs
+++ b/rust/bindings/uniffi/ensu/src/llm.rs
@@ -110,6 +110,16 @@ impl LlmModel {
Ok(Arc::new(LlmContext { handle }))
}
+ pub fn new_title_context(
+ &self,
+ chat_context_size: u32,
+ n_threads: Option<i32>,
+ ) -> Result<Arc<LlmContext>, LlmError> {
+ let handle = llm::Context::new_title(&self.handle, chat_context_size, n_threads)
+ .map_err(LlmError::from)?;
+ Ok(Arc::new(LlmContext { handle }))
+ }
+
pub fn new_embedding_context(
&self,
n_threads: Option<i32>,
diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rs
index c7b9374ca6..ce2b0f2501 100644
--- a/rust/crates/ensu/src/config.rs
+++ b/rust/crates/ensu/src/config.rs
@@ -3,11 +3,10 @@ mod knowledge;
pub use knowledge::knowledge_dataset;
pub use knowledge::{AttributionConfig, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig};
pub(crate) use knowledge::{
- EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL,
- KNOWLEDGE_LICENSE_URL, KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE,
- KNOWLEDGE_VECTORS_FILE, format_knowledge_document_prompt, is_path_safe_component,
- knowledge_artifact_urls, knowledge_datasets, knowledge_embedding_config,
- knowledge_index_contract,
+ KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL, KNOWLEDGE_LICENSE_URL,
+ KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,
+ format_knowledge_document_prompt, is_path_safe_component, knowledge_artifact_urls,
+ knowledge_datasets, knowledge_embedding_config, knowledge_index_contract,
};
#[derive(Debug, Clone)]
diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rs
index b3d91eb520..ab6da1b1fb 100644
--- a/rust/crates/ensu/src/config/knowledge.rs
+++ b/rust/crates/ensu/src/config/knowledge.rs
@@ -22,7 +22,7 @@ const EMBEDDING_MODEL_SHA256: &str =
const EMBEDDING_TARGET_ID: &str = "embeddinggemma-300m-q8-0";
const SOURCE_DIM: u32 = 768;
const RETRIEVAL_DIM: u32 = 512;
-pub(crate) const EMBEDDING_CONTEXT_SIZE: u32 = 2_048;
+const EMBEDDING_CONTEXT_SIZE: u32 = 2_048;
const MAX_HITS: u32 = 2;
const MAX_CONTEXT_UTF8_BYTES: u32 = 6_000;
const MODIFICATION_NOTICE: &str = "Adapted by Ente";
diff --git a/rust/crates/ensu/src/llm/context.rs b/rust/crates/ensu/src/llm/context.rs
index e0cd62b1b7..468a33a941 100644
--- a/rust/crates/ensu/src/llm/context.rs
+++ b/rust/crates/ensu/src/llm/context.rs
@@ -45,6 +45,17 @@ impl Context {
Self::spawn(move || LocalContext::new(&model, params))
}
+ pub fn new_title(
+ model: &ModelRef,
+ chat_context_size: u32,
+ n_threads: Option<i32>,
+ ) -> Result<ContextRef, Error> {
+ Self::new(
+ model,
+ super::memory::title_context_params(chat_context_size, n_threads),
+ )
+ }
+
pub fn new_embedding(
model: &ModelRef,
params: EmbeddingContextParams,
diff --git a/rust/crates/ensu/src/llm/memory.rs b/rust/crates/ensu/src/llm/memory.rs
index 8e52ab64f1..8a2d06606c 100644
--- a/rust/crates/ensu/src/llm/memory.rs
+++ b/rust/crates/ensu/src/llm/memory.rs
@@ -1,4 +1,5 @@
-use crate::config::{EMBEDDING_CONTEXT_SIZE, ModelRuntimeSurface};
+use super::ContextParams;
+use crate::config::ModelRuntimeSurface;
const MIB: u64 = 1024 * 1024;
const MEMORY_RESERVE_BYTES: u64 = 256 * MIB;
@@ -35,8 +36,7 @@ pub fn memory_budget(
loaded_chat: Option<&ChatModelMemory>,
query_bytes: Option<u64>,
) -> MemoryBudget {
- let token_bound = query_bytes
- .map(|bytes| bytes.min(u64::from(EMBEDDING_CONTEXT_SIZE)) + EMBEDDING_QUERY_OVERHEAD);
+ let token_bound = query_bytes.map(|bytes| bytes.saturating_add(EMBEDDING_QUERY_OVERHEAD));
let short_query = token_bound.is_some_and(|tokens| tokens <= 512);
if matches!(surface, ModelRuntimeSurface::Android)
&& matches!(operation, MemoryOperation::Embedding)
@@ -60,18 +60,36 @@ pub fn memory_budget(
let additional_bytes = match operation {
MemoryOperation::Embedding if short_query => 1024 * MIB,
MemoryOperation::Embedding => 2560 * MIB,
- MemoryOperation::Voice => 2048 * MIB,
+ MemoryOperation::Voice => match surface {
+ ModelRuntimeSurface::Android | ModelRuntimeSurface::Ios => 1280 * MIB,
+ ModelRuntimeSurface::Desktop => 2048 * MIB,
+ },
MemoryOperation::Title => 512 * MIB,
};
MemoryBudget {
required_bytes: additional_bytes + MEMORY_RESERVE_BYTES,
- system_reserve_cap_bytes: None,
+ system_reserve_cap_bytes: (matches!(surface, ModelRuntimeSurface::Android)
+ && matches!(operation, MemoryOperation::Voice))
+ .then_some(512 * MIB),
+ }
+}
+
+pub(super) fn title_context_params(
+ chat_context_size: u32,
+ n_threads: Option<i32>,
+) -> ContextParams {
+ ContextParams {
+ context_size: Some(chat_context_size.min(1024) as i32),
+ n_threads,
+ n_batch: Some(128),
}
}
#[cfg(test)]
mod tests {
use super::*;
+ use MemoryOperation::{Embedding, Title, Voice};
+ use ModelRuntimeSurface::{Android, Desktop, Ios};
#[test]
fn required_work_preserves_minimum_reserve() {
@@ -81,19 +99,29 @@ mod tests {
}
#[test]
- fn generic_budgets_cover_query_boundaries_and_required_reserve() {
- for (operation, query_bytes, required_mib) in [
- (MemoryOperation::Embedding, None, 2816),
- (MemoryOperation::Embedding, Some(0), 1280),
- (MemoryOperation::Embedding, Some(448), 1280),
- (MemoryOperation::Embedding, Some(449), 2816),
- (MemoryOperation::Embedding, Some(u64::MAX), 2816),
- (MemoryOperation::Voice, None, 2304),
- (MemoryOperation::Title, None, 768),
+ fn title_context_caps_memory_without_raising_the_chat_limit() {
+ for (chat_context_size, expected) in [(512, 512), (4096, 1024)] {
+ let params = title_context_params(chat_context_size, Some(2));
+ assert_eq!(params.context_size, Some(expected));
+ assert_eq!(params.n_batch, Some(128));
+ }
+ }
+
+ #[test]
+ fn generic_budgets_cover_query_boundaries_and_platform_reserves() {
+ for (surface, operation, query_bytes, required_mib, reserve_cap) in [
+ (Android, Embedding, None, 2816, None),
+ (Android, Embedding, Some(448), 1280, None),
+ (Android, Embedding, Some(449), 2816, None),
+ (Android, Embedding, Some(u64::MAX), 2816, None),
+ (Android, Title, None, 768, None),
+ (Android, Voice, None, 1536, Some(512 * MIB)),
+ (Ios, Voice, None, 1536, None),
+ (Desktop, Voice, None, 2304, None),
] {
- let budget = memory_budget(ModelRuntimeSurface::Android, operation, None, query_bytes);
+ let budget = memory_budget(surface, operation, None, query_bytes);
assert_eq!(budget.required_bytes, required_mib * MIB);
- assert_eq!(budget.system_reserve_cap_bytes, None);
+ assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);
}
}
@@ -105,69 +133,41 @@ mod tests {
context_size: 12032,
};
for (query_bytes, required_mib, reserve_cap) in [
- (Some(0), 1024, Some(512 * MIB)),
(Some(192), 1024, Some(512 * MIB)),
(Some(193), 1280, Some(512 * MIB)),
(Some(448), 1280, Some(512 * MIB)),
(Some(449), 2816, None),
- (Some(u64::MAX), 2816, None),
(None, 2816, None),
] {
- let budget = memory_budget(
- ModelRuntimeSurface::Android,
- MemoryOperation::Embedding,
- Some(&chat),
- query_bytes,
- );
+ let budget = memory_budget(Android, Embedding, Some(&chat), query_bytes);
assert_eq!(budget.required_bytes, required_mib * MIB);
assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);
}
- for (operation, required_mib) in [
- (MemoryOperation::Voice, 2304),
- (MemoryOperation::Title, 768),
- ] {
- let budget = memory_budget(
- ModelRuntimeSurface::Android,
- operation,
- Some(&chat),
- Some(0),
- );
+ for (operation, required_mib, reserve_cap) in
+ [(Voice, 1536, Some(512 * MIB)), (Title, 768, None)]
+ {
+ let budget = memory_budget(Android, operation, Some(&chat), Some(0));
assert_eq!(budget.required_bytes, required_mib * MIB);
- assert_eq!(budget.system_reserve_cap_bytes, None);
+ assert_eq!(budget.system_reserve_cap_bytes, reserve_cap);
}
}
#[test]
fn unmeasured_platforms_models_and_contexts_keep_generic_budgets() {
for (surface, model_id, model_bytes, context_size) in [
- (ModelRuntimeSurface::Ios, "lfm-vl-1.6b", 695_752_480, 4096),
- (
- ModelRuntimeSurface::Desktop,
- "lfm-vl-1.6b",
- 695_752_480,
- 4096,
- ),
- (ModelRuntimeSurface::Android, "gemma", 695_752_480, 4096),
- (
- ModelRuntimeSurface::Android,
- "lfm-vl-1.6b",
- 695_752_481,
- 4096,
- ),
- (ModelRuntimeSurface::Android, "lfm-vl-1.6b", 695_752_480, 0),
- (
- ModelRuntimeSurface::Android,
- "lfm-vl-1.6b",
- 695_752_480,
- 12033,
- ),
+ (Ios, "lfm-vl-1.6b", 695_752_480, 4096),
+ (Desktop, "lfm-vl-1.6b", 695_752_480, 4096),
+ (Android, "gemma", 695_752_480, 4096),
+ (Android, "lfm-vl-1.6b", 695_752_481, 4096),
+ (Android, "lfm-vl-1.6b", 695_752_480, 0),
+ (Android, "lfm-vl-1.6b", 695_752_480, 12033),
] {
let chat = ChatModelMemory {
model_id: model_id.into(),
model_bytes,
context_size,
};
- let budget = memory_budget(surface, MemoryOperation::Embedding, Some(&chat), Some(128));
+ let budget = memory_budget(surface, Embedding, Some(&chat), Some(128));
assert_eq!(budget.required_bytes, 1280 * MIB);
assert_eq!(budget.system_reserve_cap_bytes, None);
}