Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Fix mobile warm-up coordination with voice tasks and recovery

setalpfaf1ad11970c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 74989294d6..36887ed8ae 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -23,6 +23,7 @@ import io.ente.ensu.notes.NotesStore
import io.ente.ensu.settings.DeveloperSettingsState
import io.ente.ensu.settings.SessionPreferencesDataStore
import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
@@ -98,6 +99,8 @@ class AppStore(
fun suppressChatWarmup() = modelSettingsActions.suppressChatWarmup()
+ fun trackVoiceInput(job: Job) = modelSettingsActions.trackVoiceInput(job)
+
fun refreshDeviceCapability(scope: CoroutineScope? = null) {
val capability = deviceCapabilityProvider.chatCapability()
val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemory
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index c992dcf1f4..ce423ca23e 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -111,7 +111,7 @@ internal fun HomeNavigation(
popExitTransition = { backExit() },
) {
ChatView(
- onVoiceInputStarted = store::suppressChatWarmup,
+ onVoiceInputJob = store::trackVoiceInput,
chatState = appState.chat,
assetStore = store.assetStore,
transcriber = store.transcriber,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index 54da62eaf9..7f34f1c20c 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -51,6 +51,7 @@ import io.ente.ensu.bindings.Transcriber
import io.ente.ensu.designsystem.EnsuColor
import io.ente.ensu.designsystem.EnsuSpacing
import io.ente.ensu.device.ChatDeviceCapability
+import kotlinx.coroutines.Job
@Composable
fun ChatView(
@@ -73,7 +74,7 @@ fun ChatView(
onDismissUnsupportedDeviceDialog: () -> Unit,
onOverflowTrim: () -> Unit,
onOverflowCancel: () -> Unit,
- onVoiceInputStarted: () -> Unit,
+ onVoiceInputJob: (Job) -> Unit,
) {
val density = LocalDensity.current
val context = LocalContext.current
@@ -90,6 +91,7 @@ fun ChatView(
rememberVoiceTranscriptionController(
assetStore = assetStore,
transcriber = transcriber,
+ onVoiceInputJob = onVoiceInputJob,
onTranscript = { transcript ->
latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))
},
@@ -118,7 +120,6 @@ fun ChatView(
pendingVoiceSessionKey = null
if (granted) {
if (requestedSessionKey != null) {
- onVoiceInputStarted()
voiceController.startRecording {
latestSessionKey == requestedSessionKey && latestCanStartVoiceInput
}
@@ -298,7 +299,6 @@ fun ChatView(
) == PackageManager.PERMISSION_GRANTED
if (canStartVoiceInput && hasMicrophonePermission) {
val requestedSessionKey = sessionKey
- onVoiceInputStarted()
voiceController.startRecording {
latestSessionKey == requestedSessionKey &&
latestCanStartVoiceInput
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index dd3500e1cf..25c79d67df 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -26,9 +26,12 @@ import io.ente.ensu.llm.withMaintenanceSuspended
import io.ente.ensu.notes.LocalNotesStore
import java.io.ByteArrayOutputStream
import java.io.IOException
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
import kotlin.coroutines.coroutineContext
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
@@ -80,12 +83,19 @@ internal fun rememberVoiceTranscriptionController(
assetStore: AssetStore,
transcriber: Transcriber,
onTranscript: (String) -> Unit,
+ onVoiceInputJob: (Job) -> Unit,
): VoiceTranscriptionController {
val lifecycleOwner = LocalLifecycleOwner.current
val notes = LocalNotesStore.current
val controller =
remember(transcriber, notes) {
- VoiceTranscriptionController(assetStore, transcriber, onTranscript, notes)
+ VoiceTranscriptionController(
+ assetStore,
+ transcriber,
+ onTranscript,
+ notes,
+ onVoiceInputJob,
+ )
}
DisposableEffect(controller, lifecycleOwner) {
val observer = LifecycleEventObserver { _, event ->
@@ -105,6 +115,7 @@ internal class VoiceTranscriptionController(
private val transcriber: Transcriber,
private val onTranscript: (String) -> Unit,
private val maintenance: ModelMaintenance?,
+ private val onVoiceInputJob: (Job) -> Unit,
) {
private val modelAssets =
listOf(
@@ -126,6 +137,15 @@ internal class VoiceTranscriptionController(
var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)
private set
+ private fun launchVoiceWork(
+ context: CoroutineContext = EmptyCoroutineContext,
+ block: suspend CoroutineScope.() -> Unit,
+ ): Job =
+ scope.launch(context, start = CoroutineStart.LAZY, block = block).also {
+ onVoiceInputJob(it)
+ it.start()
+ }
+
fun onPermissionDenied() {
state = VoiceInputState.Error("Microphone permission is required for voice input.")
}
@@ -137,14 +157,14 @@ internal class VoiceTranscriptionController(
return
}
- preparingRecordingJob = scope.launch {
+ preparingRecordingJob = launchVoiceWork {
try {
ensureTranscriptionModelDownloaded()
if (!isActive || !canStartRecording()) {
if (state is VoiceInputState.Downloading) {
state = VoiceInputState.Idle
}
- return@launch
+ return@launchVoiceWork
}
beginRecording()
preloadTranscriptionModel()
@@ -189,7 +209,7 @@ internal class VoiceTranscriptionController(
state = VoiceInputState.Recording
recordingJob =
- scope.launch(Dispatchers.IO) {
+ launchVoiceWork(Dispatchers.IO) {
var recorder: AudioRecord? = null
try {
val activeRecorder = createRecorder(recordingSampleRate)
@@ -239,14 +259,14 @@ internal class VoiceTranscriptionController(
state = VoiceInputState.Transcribing
preparingRecordingJob?.cancel()
preparingRecordingJob = null
- scope.launch {
+ launchVoiceWork {
recordingJob?.cancelAndJoin()
recordingJob = null
val pcm = synchronized(bufferLock) { recordedPcm.toByteArray() }
if (pcm.size < minimumRecordingBytes(recordingSampleRate)) {
showTransientError("No speech captured.")
- return@launch
+ return@launchVoiceWork
}
transcribeRecording(pcm, recordingSampleRate)
@@ -335,7 +355,7 @@ internal class VoiceTranscriptionController(
private fun preloadTranscriptionModel() {
transcriptionPreloadJob?.cancel()
transcriptionPreloadJob =
- scope.launch(Dispatchers.IO) {
+ launchVoiceWork(Dispatchers.IO) {
try {
maintenance.withMaintenanceSuspended { transcriber.loadModel() }
} catch (error: CancellationException) {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index 23293932d9..bc37e2affc 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -15,6 +15,7 @@ import io.ente.ensu.notes.NotesStore
import io.ente.ensu.settings.IS_ENSU_PACKS_ENABLED
import io.ente.ensu.settings.SessionPreferencesDataStore
import java.util.UUID
+import java.util.concurrent.atomic.AtomicInteger
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
@@ -36,6 +37,7 @@ internal class ModelSettingsActions(
private var scope: CoroutineScope? = null
private var modelDownloadJob: Job? = null
private var chatActive = false
+ private val activeVoiceJobs = AtomicInteger()
private var warmupSuppressed = false
private var warmupJob: Job? = null
private var warmupOwner: String? = null
@@ -332,6 +334,14 @@ internal class ModelSettingsActions(
cancelChatWarmup()
}
+ fun trackVoiceInput(job: Job) {
+ activeVoiceJobs.incrementAndGet()
+ suppressChatWarmup()
+ job.invokeOnCompletion {
+ if (activeVoiceJobs.decrementAndGet() == 0) scope?.launch { refreshChatWarmup() }
+ }
+ }
+
private fun cancelChatWarmup() {
warmupJob?.cancel()
warmupJob = null
@@ -347,6 +357,7 @@ internal class ModelSettingsActions(
val eligible =
chatActive &&
!warmupSuppressed &&
+ activeVoiceJobs.get() == 0 &&
current.chat.deviceCapability.isChatSupported() &&
current.knowledge.packs.values.none { it.enabled } &&
notesStore.state.value.collections.isEmpty()
@@ -373,7 +384,8 @@ internal class ModelSettingsActions(
coroutineContext.ensureActive()
val latest = state.value
if (
- latest.chat.isGenerating ||
+ activeVoiceJobs.get() > 0 ||
+ latest.chat.isGenerating ||
latest.chat.isDownloading ||
latest.knowledge.packs.values.any { it.enabled } ||
notesStore.state.value.collections.isNotEmpty()
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index e5370b52c1..2e5bea7bd1 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -223,6 +223,14 @@ final class ChatViewModel: ObservableObject {
loadMessagesFromDb(for: current)
}
+ voiceTranscriber.onTaskActivityChanged = { [weak self] active in
+ guard let self else { return }
+ if active {
+ self.suppressChatWarmup()
+ } else {
+ self.refreshChatWarmup()
+ }
+ }
refreshDeviceCapability()
refreshModelDownloadInfo()
warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)
@@ -263,7 +271,7 @@ final class ChatViewModel: ObservableObject {
let key = modelReadyKey(for: selection)
let eligible =
chatActive && !warmupSuppressed && !isChatUnsupported
- && !voiceInputState.isWorking
+ && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks
&& knowledgeStore.packs.allSatisfy { !$0.enabled } && notesStore.collections.isEmpty
guard eligible else {
cancelChatWarmup()
@@ -283,11 +291,23 @@ final class ChatViewModel: ObservableObject {
await notesStore.bootstrap()
try Task.checkCancellation()
guard !isGenerating, !isDownloading, !voiceInputState.isWorking,
+ !voiceTranscriber.hasActiveTasks,
knowledgeStore.packs.allSatisfy({ !$0.enabled }), notesStore.collections.isEmpty
else { return }
try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)
} catch {
- if !isCancellation(error) { logger.info("Chat warm-up skipped") }
+ guard !Task.isCancelled, !isCancellation(error), warmupAttempt?.owner == owner,
+ modelReadyKey(for: modelSettings.currentSelection()) == key,
+ !isGenerating, !isDownloading
+ else { return }
+ logger.error("Model load failed", error)
+ downloadToast = DownloadToastState(
+ phase: .errorLoad,
+ percent: nil,
+ status: userFacingModelReadyError(error, wasDownloaded: true),
+ offerRetryDownload: true
+ )
+ isModelDownloaded = false
}
}
}
@@ -500,13 +520,12 @@ final class ChatViewModel: ObservableObject {
}
func cancelVoiceInput() {
- voiceTransientErrorTask?.cancel()
- voiceTransientErrorTask = nil
voiceTranscriber.cancel()
- voiceInputState = .idle
+ setVoiceInputState(.idle)
}
private func setVoiceInputState(_ state: VoiceInputState) {
+ defer { refreshChatWarmup() }
voiceTransientErrorTask?.cancel()
voiceTransientErrorTask = nil
voiceInputState = state
@@ -885,7 +904,9 @@ final class ChatViewModel: ObservableObject {
}
}
- private func ensureRequiredModelsReadyShared(_ selection: LlmModelSelection) async throws {
+ private func ensureRequiredModelsReadyShared(
+ _ selection: LlmModelSelection, recoverDownloadedModel: Bool
+ ) async throws {
if isChatUnsupported {
throw UnsupportedDeviceMemoryError(capability: deviceCapability)
}
@@ -902,6 +923,10 @@ final class ChatViewModel: ObservableObject {
let taskId = UUID()
let task = Task {
+ if recoverDownloadedModel {
+ try await self.ensureChatModelReady(selection)
+ return
+ }
var retryCount = 0
while true {
do {
@@ -953,21 +978,25 @@ final class ChatViewModel: ObservableObject {
}
let selection = modelSettings.currentSelection()
- if provider.isModelDownloaded(selection) {
- isModelDownloaded = true
+ let wasDownloaded = provider.isModelDownloaded(selection)
+ if wasDownloaded && isModelDownloaded {
modelDownloadSizeBytes = nil
return
}
+ cancelChatWarmup()
isDownloading = true
seedDownloadProgressMemory()
- modelDownloadLoggedStart = true
- logger.info("Model download started", details: "model=\(selection.id)")
+ modelDownloadLoggedStart = !wasDownloaded
+ if modelDownloadLoggedStart {
+ logger.info("Model download started", details: "model=\(selection.id)")
+ }
modelDownloadTask?.cancel()
modelDownloadTask = Task {
do {
- try await self.ensureRequiredModelsReadyShared(selection)
+ try await self.ensureRequiredModelsReadyShared(
+ selection, recoverDownloadedModel: wasDownloaded)
self.handleProgress(DownloadProgress(percent: 100, status: "Ready", phase: .ready))
} catch {
if isCancellation(error) {
@@ -981,9 +1010,9 @@ final class ChatViewModel: ObservableObject {
}
await MainActor.run {
self.downloadToast = DownloadToastState(
- phase: .errorDownload,
+ phase: wasDownloaded ? .errorLoad : .errorDownload,
percent: nil,
- status: self.userFacingModelReadyError(error, wasDownloaded: false),
+ status: self.userFacingModelReadyError(error, wasDownloaded: wasDownloaded),
offerRetryDownload: true
)
self.isDownloading = false
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index a107605fd0..6bfd28ce42 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -83,6 +83,7 @@ final class VoiceTranscriptionService {
typealias TranscriptHandler = @MainActor @Sendable (String) -> Void
weak var modelMaintenance: (any ModelMaintenance)?
+ var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?
private let transcriber: Transcriber
private let assetStore: AssetStore
private let modelAssets: [Asset]
@@ -90,6 +91,9 @@ final class VoiceTranscriptionService {
private var preloadTask: Task<Void, Never>?
private var activeVoiceTaskId = UUID()
private var activeDownloadId: UUID?
+ private var activeTaskCount = 0
+
+ var hasActiveTasks: Bool { activeTaskCount > 0 }
private let recorder = PcmAudioRecorder()
@@ -99,6 +103,21 @@ final class VoiceTranscriptionService {
self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]
}
+ private func launchVoiceTask(
+ priority: TaskPriority, operation: @escaping @Sendable () async -> Void
+ ) -> Task<Void, Never> {
+ activeTaskCount += 1
+ if activeTaskCount == 1 { onTaskActivityChanged?(true) }
+ let task = Task.detached(priority: priority, operation: operation)
+ Task { [weak self] in
+ await task.value
+ guard let self else { return }
+ self.activeTaskCount -= 1
+ if self.activeTaskCount == 0 { self.onTaskActivityChanged?(false) }
+ }
+ return task
+ }
+
func startRecording(
onState: @escaping StateHandler,
shouldStartRecording: @escaping @MainActor @Sendable () -> Bool = { true }
@@ -152,7 +171,7 @@ final class VoiceTranscriptionService {
let maintenance = modelMaintenance
let maintenanceScope = maintenance?.suspendMaintenance()
- transcriptionTask = Task.detached(priority: .userInitiated) { [weak self] in
+ transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
defer { maintenanceScope?.close() }
await maintenance?.awaitMaintenance()
do {
@@ -226,7 +245,7 @@ final class VoiceTranscriptionService {
let maintenance = modelMaintenance
let maintenanceScope = maintenance?.suspendMaintenance()
- transcriptionTask = Task.detached(priority: .userInitiated) { [weak self] in
+ transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
defer { maintenanceScope?.close() }
await maintenance?.awaitMaintenance()
do {
@@ -301,7 +320,7 @@ final class VoiceTranscriptionService {
preloadTask?.cancel()
let maintenance = modelMaintenance
let maintenanceScope = maintenance?.suspendMaintenance()
- preloadTask = Task.detached(priority: .utility) {
+ preloadTask = launchVoiceTask(priority: .utility) {
defer { maintenanceScope?.close() }
await maintenance?.awaitMaintenance()
do {