π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix mobile warm-up coordination with voice tasks and recovery
setalpfaf1ad11970c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 74989294d6..36887ed8ae 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -23,6 +23,7 @@ import io.ente.ensu.notes.NotesStoreimport io.ente.ensu.settings.DeveloperSettingsStateimport io.ente.ensu.settings.SessionPreferencesDataStoreimport kotlinx.coroutines.CoroutineScope+import kotlinx.coroutines.Jobimport kotlinx.coroutines.flow.MutableStateFlowimport kotlinx.coroutines.flow.StateFlowimport kotlinx.coroutines.flow.asStateFlow@@ -98,6 +99,8 @@ class AppStore(fun suppressChatWarmup() = modelSettingsActions.suppressChatWarmup()+ fun trackVoiceInput(job: Job) = modelSettingsActions.trackVoiceInput(job)+fun refreshDeviceCapability(scope: CoroutineScope? = null) {val capability = deviceCapabilityProvider.chatCapability()val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemorydiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex c992dcf1f4..ce423ca23e 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -111,7 +111,7 @@ internal fun HomeNavigation(popExitTransition = { backExit() },) {ChatView(- onVoiceInputStarted = store::suppressChatWarmup,+ onVoiceInputJob = store::trackVoiceInput,chatState = appState.chat,assetStore = store.assetStore,transcriber = store.transcriber,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex 54da62eaf9..7f34f1c20c 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -51,6 +51,7 @@ import io.ente.ensu.bindings.Transcriberimport io.ente.ensu.designsystem.EnsuColorimport io.ente.ensu.designsystem.EnsuSpacingimport io.ente.ensu.device.ChatDeviceCapability+import kotlinx.coroutines.Job@Composablefun ChatView(@@ -73,7 +74,7 @@ fun ChatView(onDismissUnsupportedDeviceDialog: () -> Unit,onOverflowTrim: () -> Unit,onOverflowCancel: () -> Unit,- onVoiceInputStarted: () -> Unit,+ onVoiceInputJob: (Job) -> Unit,) {val density = LocalDensity.currentval context = LocalContext.current@@ -90,6 +91,7 @@ fun ChatView(rememberVoiceTranscriptionController(assetStore = assetStore,transcriber = transcriber,+ onVoiceInputJob = onVoiceInputJob,onTranscript = { transcript ->latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))},@@ -118,7 +120,6 @@ fun ChatView(pendingVoiceSessionKey = nullif (granted) {if (requestedSessionKey != null) {- onVoiceInputStarted()voiceController.startRecording {latestSessionKey == requestedSessionKey && latestCanStartVoiceInput}@@ -298,7 +299,6 @@ fun ChatView() == PackageManager.PERMISSION_GRANTEDif (canStartVoiceInput && hasMicrophonePermission) {val requestedSessionKey = sessionKey- onVoiceInputStarted()voiceController.startRecording {latestSessionKey == requestedSessionKey &&latestCanStartVoiceInputdiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex dd3500e1cf..25c79d67df 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -26,9 +26,12 @@ import io.ente.ensu.llm.withMaintenanceSuspendedimport io.ente.ensu.notes.LocalNotesStoreimport java.io.ByteArrayOutputStreamimport java.io.IOException+import kotlin.coroutines.CoroutineContext+import kotlin.coroutines.EmptyCoroutineContextimport kotlin.coroutines.coroutineContextimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.CoroutineScope+import kotlinx.coroutines.CoroutineStartimport kotlinx.coroutines.Dispatchersimport kotlinx.coroutines.Jobimport kotlinx.coroutines.SupervisorJob@@ -80,12 +83,19 @@ internal fun rememberVoiceTranscriptionController(assetStore: AssetStore,transcriber: Transcriber,onTranscript: (String) -> Unit,+ onVoiceInputJob: (Job) -> Unit,): VoiceTranscriptionController {val lifecycleOwner = LocalLifecycleOwner.currentval notes = LocalNotesStore.currentval controller =remember(transcriber, notes) {- VoiceTranscriptionController(assetStore, transcriber, onTranscript, notes)+ VoiceTranscriptionController(+ assetStore,+ transcriber,+ onTranscript,+ notes,+ onVoiceInputJob,+ )}DisposableEffect(controller, lifecycleOwner) {val observer = LifecycleEventObserver { _, event ->@@ -105,6 +115,7 @@ internal class VoiceTranscriptionController(private val transcriber: Transcriber,private val onTranscript: (String) -> Unit,private val maintenance: ModelMaintenance?,+ private val onVoiceInputJob: (Job) -> Unit,) {private val modelAssets =listOf(@@ -126,6 +137,15 @@ internal class VoiceTranscriptionController(var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)private set+ private fun launchVoiceWork(+ context: CoroutineContext = EmptyCoroutineContext,+ block: suspend CoroutineScope.() -> Unit,+ ): Job =+ scope.launch(context, start = CoroutineStart.LAZY, block = block).also {+ onVoiceInputJob(it)+ it.start()+ }+fun onPermissionDenied() {state = VoiceInputState.Error("Microphone permission is required for voice input.")}@@ -137,14 +157,14 @@ internal class VoiceTranscriptionController(return}- preparingRecordingJob = scope.launch {+ preparingRecordingJob = launchVoiceWork {try {ensureTranscriptionModelDownloaded()if (!isActive || !canStartRecording()) {if (state is VoiceInputState.Downloading) {state = VoiceInputState.Idle}- return@launch+ return@launchVoiceWork}beginRecording()preloadTranscriptionModel()@@ -189,7 +209,7 @@ internal class VoiceTranscriptionController(state = VoiceInputState.RecordingrecordingJob =- scope.launch(Dispatchers.IO) {+ launchVoiceWork(Dispatchers.IO) {var recorder: AudioRecord? = nulltry {val activeRecorder = createRecorder(recordingSampleRate)@@ -239,14 +259,14 @@ internal class VoiceTranscriptionController(state = VoiceInputState.TranscribingpreparingRecordingJob?.cancel()preparingRecordingJob = null- scope.launch {+ launchVoiceWork {recordingJob?.cancelAndJoin()recordingJob = nullval pcm = synchronized(bufferLock) { recordedPcm.toByteArray() }if (pcm.size < minimumRecordingBytes(recordingSampleRate)) {showTransientError("No speech captured.")- return@launch+ return@launchVoiceWork}transcribeRecording(pcm, recordingSampleRate)@@ -335,7 +355,7 @@ internal class VoiceTranscriptionController(private fun preloadTranscriptionModel() {transcriptionPreloadJob?.cancel()transcriptionPreloadJob =- scope.launch(Dispatchers.IO) {+ launchVoiceWork(Dispatchers.IO) {try {maintenance.withMaintenanceSuspended { transcriber.loadModel() }} catch (error: CancellationException) {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex 23293932d9..bc37e2affc 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -15,6 +15,7 @@ import io.ente.ensu.notes.NotesStoreimport io.ente.ensu.settings.IS_ENSU_PACKS_ENABLEDimport io.ente.ensu.settings.SessionPreferencesDataStoreimport java.util.UUID+import java.util.concurrent.atomic.AtomicIntegerimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.CoroutineScopeimport kotlinx.coroutines.Job@@ -36,6 +37,7 @@ internal class ModelSettingsActions(private var scope: CoroutineScope? = nullprivate var modelDownloadJob: Job? = nullprivate var chatActive = false+ private val activeVoiceJobs = AtomicInteger()private var warmupSuppressed = falseprivate var warmupJob: Job? = nullprivate var warmupOwner: String? = null@@ -332,6 +334,14 @@ internal class ModelSettingsActions(cancelChatWarmup()}+ fun trackVoiceInput(job: Job) {+ activeVoiceJobs.incrementAndGet()+ suppressChatWarmup()+ job.invokeOnCompletion {+ if (activeVoiceJobs.decrementAndGet() == 0) scope?.launch { refreshChatWarmup() }+ }+ }+private fun cancelChatWarmup() {warmupJob?.cancel()warmupJob = null@@ -347,6 +357,7 @@ internal class ModelSettingsActions(val eligible =chatActive &&!warmupSuppressed &&+ activeVoiceJobs.get() == 0 &¤t.chat.deviceCapability.isChatSupported() &¤t.knowledge.packs.values.none { it.enabled } &¬esStore.state.value.collections.isEmpty()@@ -373,7 +384,8 @@ internal class ModelSettingsActions(coroutineContext.ensureActive()val latest = state.valueif (- latest.chat.isGenerating ||+ activeVoiceJobs.get() > 0 ||+ latest.chat.isGenerating ||latest.chat.isDownloading ||latest.knowledge.packs.values.any { it.enabled } ||notesStore.state.value.collections.isNotEmpty()diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex e5370b52c1..2e5bea7bd1 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -223,6 +223,14 @@ final class ChatViewModel: ObservableObject {loadMessagesFromDb(for: current)}+ voiceTranscriber.onTaskActivityChanged = { [weak self] active in+ guard let self else { return }+ if active {+ self.suppressChatWarmup()+ } else {+ self.refreshChatWarmup()+ }+ }refreshDeviceCapability()refreshModelDownloadInfo()warmupObservation = Publishers.CombineLatest(knowledgeStore.$packs, notesStore.$collections)@@ -263,7 +271,7 @@ final class ChatViewModel: ObservableObject {let key = modelReadyKey(for: selection)let eligible =chatActive && !warmupSuppressed && !isChatUnsupported- && !voiceInputState.isWorking+ && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks&& knowledgeStore.packs.allSatisfy { !$0.enabled } && notesStore.collections.isEmptyguard eligible else {cancelChatWarmup()@@ -283,11 +291,23 @@ final class ChatViewModel: ObservableObject {await notesStore.bootstrap()try Task.checkCancellation()guard !isGenerating, !isDownloading, !voiceInputState.isWorking,+ !voiceTranscriber.hasActiveTasks,knowledgeStore.packs.allSatisfy({ !$0.enabled }), notesStore.collections.isEmptyelse { return }try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)} catch {- if !isCancellation(error) { logger.info("Chat warm-up skipped") }+ guard !Task.isCancelled, !isCancellation(error), warmupAttempt?.owner == owner,+ modelReadyKey(for: modelSettings.currentSelection()) == key,+ !isGenerating, !isDownloading+ else { return }+ logger.error("Model load failed", error)+ downloadToast = DownloadToastState(+ phase: .errorLoad,+ percent: nil,+ status: userFacingModelReadyError(error, wasDownloaded: true),+ offerRetryDownload: true+ )+ isModelDownloaded = false}}}@@ -500,13 +520,12 @@ final class ChatViewModel: ObservableObject {}func cancelVoiceInput() {- voiceTransientErrorTask?.cancel()- voiceTransientErrorTask = nilvoiceTranscriber.cancel()- voiceInputState = .idle+ setVoiceInputState(.idle)}private func setVoiceInputState(_ state: VoiceInputState) {+ defer { refreshChatWarmup() }voiceTransientErrorTask?.cancel()voiceTransientErrorTask = nilvoiceInputState = state@@ -885,7 +904,9 @@ final class ChatViewModel: ObservableObject {}}- private func ensureRequiredModelsReadyShared(_ selection: LlmModelSelection) async throws {+ private func ensureRequiredModelsReadyShared(+ _ selection: LlmModelSelection, recoverDownloadedModel: Bool+ ) async throws {if isChatUnsupported {throw UnsupportedDeviceMemoryError(capability: deviceCapability)}@@ -902,6 +923,10 @@ final class ChatViewModel: ObservableObject {let taskId = UUID()let task = Task {+ if recoverDownloadedModel {+ try await self.ensureChatModelReady(selection)+ return+ }var retryCount = 0while true {do {@@ -953,21 +978,25 @@ final class ChatViewModel: ObservableObject {}let selection = modelSettings.currentSelection()- if provider.isModelDownloaded(selection) {- isModelDownloaded = true+ let wasDownloaded = provider.isModelDownloaded(selection)+ if wasDownloaded && isModelDownloaded {modelDownloadSizeBytes = nilreturn}+ cancelChatWarmup()isDownloading = trueseedDownloadProgressMemory()- modelDownloadLoggedStart = true- logger.info("Model download started", details: "model=\(selection.id)")+ modelDownloadLoggedStart = !wasDownloaded+ if modelDownloadLoggedStart {+ logger.info("Model download started", details: "model=\(selection.id)")+ }modelDownloadTask?.cancel()modelDownloadTask = Task {do {- try await self.ensureRequiredModelsReadyShared(selection)+ try await self.ensureRequiredModelsReadyShared(+ selection, recoverDownloadedModel: wasDownloaded)self.handleProgress(DownloadProgress(percent: 100, status: "Ready", phase: .ready))} catch {if isCancellation(error) {@@ -981,9 +1010,9 @@ final class ChatViewModel: ObservableObject {}await MainActor.run {self.downloadToast = DownloadToastState(- phase: .errorDownload,+ phase: wasDownloaded ? .errorLoad : .errorDownload,percent: nil,- status: self.userFacingModelReadyError(error, wasDownloaded: false),+ status: self.userFacingModelReadyError(error, wasDownloaded: wasDownloaded),offerRetryDownload: true)self.isDownloading = falsediff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex a107605fd0..6bfd28ce42 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -83,6 +83,7 @@ final class VoiceTranscriptionService {typealias TranscriptHandler = @MainActor @Sendable (String) -> Voidweak var modelMaintenance: (any ModelMaintenance)?+ var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?private let transcriber: Transcriberprivate let assetStore: AssetStoreprivate let modelAssets: [Asset]@@ -90,6 +91,9 @@ final class VoiceTranscriptionService {private var preloadTask: Task<Void, Never>?private var activeVoiceTaskId = UUID()private var activeDownloadId: UUID?+ private var activeTaskCount = 0++ var hasActiveTasks: Bool { activeTaskCount > 0 }private let recorder = PcmAudioRecorder()@@ -99,6 +103,21 @@ final class VoiceTranscriptionService {self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]}+ private func launchVoiceTask(+ priority: TaskPriority, operation: @escaping @Sendable () async -> Void+ ) -> Task<Void, Never> {+ activeTaskCount += 1+ if activeTaskCount == 1 { onTaskActivityChanged?(true) }+ let task = Task.detached(priority: priority, operation: operation)+ Task { [weak self] in+ await task.value+ guard let self else { return }+ self.activeTaskCount -= 1+ if self.activeTaskCount == 0 { self.onTaskActivityChanged?(false) }+ }+ return task+ }+func startRecording(onState: @escaping StateHandler,shouldStartRecording: @escaping @MainActor @Sendable () -> Bool = { true }@@ -152,7 +171,7 @@ final class VoiceTranscriptionService {let maintenance = modelMaintenancelet maintenanceScope = maintenance?.suspendMaintenance()- transcriptionTask = Task.detached(priority: .userInitiated) { [weak self] in+ transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] indefer { maintenanceScope?.close() }await maintenance?.awaitMaintenance()do {@@ -226,7 +245,7 @@ final class VoiceTranscriptionService {let maintenance = modelMaintenancelet maintenanceScope = maintenance?.suspendMaintenance()- transcriptionTask = Task.detached(priority: .userInitiated) { [weak self] in+ transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] indefer { maintenanceScope?.close() }await maintenance?.awaitMaintenance()do {@@ -301,7 +320,7 @@ final class VoiceTranscriptionService {preloadTask?.cancel()let maintenance = modelMaintenancelet maintenanceScope = maintenance?.suspendMaintenance()- preloadTask = Task.detached(priority: .utility) {+ preloadTask = launchVoiceTask(priority: .utility) {defer { maintenanceScope?.close() }await maintenance?.awaitMaintenance()do {