💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Add inline mobile dictation with faster startup
setalp79673334d053
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex e692e7d94d..f5822f52d1 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -1,13 +1,16 @@package io.ente.ensuimport android.content.Context+import android.icu.text.BreakIteratorimport io.ente.ensu.assets.AssetStoreimport io.ente.ensu.bindings.ConfigDefaults+import io.ente.ensu.bindings.DictationInsertionimport io.ente.ensu.chat.Attachmentimport io.ente.ensu.chat.AttachmentStoreActionsimport io.ente.ensu.chat.ChatMessageimport io.ente.ensu.chat.ChatRepositoryimport io.ente.ensu.chat.ChatStoreActions+import io.ente.ensu.chat.VoiceTranscriptionControllerimport io.ente.ensu.device.AndroidDeviceCapabilityProviderimport io.ente.ensu.device.ChatDeviceCapabilityimport io.ente.ensu.knowledge.KnowledgeProvider@@ -21,6 +24,8 @@ import io.ente.ensu.logging.LogLevelimport io.ente.ensu.notes.NotesStoreimport io.ente.ensu.settings.DeveloperSettingsStateimport io.ente.ensu.settings.SessionPreferencesDataStore+import java.util.Locale+import java.util.UUIDimport kotlinx.coroutines.CoroutineScopeimport kotlinx.coroutines.Jobimport kotlinx.coroutines.flow.MutableStateFlow@@ -84,6 +89,60 @@ class AppStore(awaitKnowledgeReady = knowledgeStore::awaitReady,)+ private var composerRevision = 0uL+ private var emptyComposerId = UUID.randomUUID().toString()+ internal val voiceController: VoiceTranscriptionController =+ VoiceTranscriptionController(+ context.applicationContext,+ assetStore,+ llmProvider,+ logRepository,+ ::trackVoiceInput,+ ) { insertion, transcript ->+ val chat = _state.value.chat+ runCatching {+ val result =+ insertion.apply(+ chat.currentSessionId ?: emptyComposerId,+ composerRevision,+ chat.messageText,+ transcript,+ true,+ false,+ )+ chatActions.updateMessageText(result.text)+ composerRevision = result.revision+ voiceController.composerCaret = result.caretUtf16.toInt()+ true+ }+ .getOrDefault(false)+ }++ internal fun openVoiceInput(+ text: String,+ selectionStart: Int,+ selectionEnd: Int,+ isComposing: Boolean,+ ) {+ val chat = _state.value.chat+ if (isComposing || chat.isGenerating || chat.isDownloading || chat.editingMessageId != null)+ return+ if (chat.messageText != text || !voiceController.canOpen) return+ if (selectionStart !in 0..selectionEnd || selectionEnd > text.length) return+ val characters = BreakIterator.getCharacterInstance(Locale.ROOT).apply { setText(text) }+ if (!characters.isBoundary(selectionStart) || !characters.isBoundary(selectionEnd)) return+ voiceController.open(+ DictationInsertion(+ chat.currentSessionId ?: emptyComposerId,+ composerRevision,+ selectionStart.toUInt(),+ selectionEnd.toUInt(),+ )+ )+ }++ fun dispose() = voiceController.dispose()+fun bootstrap(scope: CoroutineScope) {notesStore.bootstrap(scope)chatActions.setScope(scope)@@ -100,11 +159,15 @@ class AppStore(modelSettingsActions.setChatActive(active)}- fun setAppForeground(active: Boolean) = modelSettingsActions.setAppForeground(active)+ fun setAppForeground(active: Boolean) {+ voiceController.setForeground(active)+ modelSettingsActions.setAppForeground(active)+ }fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)fun handleMemoryPressure() {+ voiceController.interrupt("Voice input paused because memory is low.")chatActions.cancelSessionSummary()modelSettingsActions.handleMemoryPressure()}@@ -173,21 +236,43 @@ class AppStore(fun createNewSession(): String = chatActions.createNewSession()- fun startNewSessionDraft() = chatActions.startNewSessionDraft()+ fun startNewSessionDraft() {+ voiceController.discard()+ emptyComposerId = UUID.randomUUID().toString()+ composerRevision+++ chatActions.startNewSessionDraft()+ }- fun selectSession(sessionId: String) = chatActions.selectSession(sessionId)+ fun selectSession(sessionId: String) {+ voiceController.discard()+ composerRevision+++ chatActions.selectSession(sessionId)+ }- fun deleteSession(sessionId: String) = chatActions.deleteSession(sessionId)+ fun deleteSession(sessionId: String) {+ if (sessionId == _state.value.chat.currentSessionId) voiceController.discard()+ chatActions.deleteSession(sessionId)+ }fun persistSelectedSession(scope: CoroutineScope, sessionId: String?) =chatActions.persistSelectedSession(scope, sessionId)- fun updateMessageText(value: String) = chatActions.updateMessageText(value)+ fun updateMessageText(value: String) {+ if (value != _state.value.chat.messageText) {+ composerRevision+++ voiceController.composerCaret = null+ }+ chatActions.updateMessageText(value)+ }fun updateBranchSelection(messageId: String, selectedIndex: Int) =chatActions.updateBranchSelection(messageId, selectedIndex)- fun beginEditing(messageId: String) = chatActions.beginEditing(messageId)+ fun beginEditing(messageId: String) {+ voiceController.discard()+ composerRevision+++ chatActions.beginEditing(messageId)+ }fun cancelEditing() = chatActions.cancelEditing()@@ -232,7 +317,12 @@ class AppStore(fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)- fun sendMessage() = chatActions.sendMessage()+ fun sendMessage() {+ if (voiceController.active) return+ voiceController.composerCaret = null+ composerRevision+++ chatActions.sendMessage()+ }fun confirmOverflowTrim() = chatActions.confirmOverflowTrim()diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.ktindex 6a57f9f65c..f775249cb9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt@@ -96,6 +96,11 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {}}+ override fun onCleared() {+ store.dispose()+ super.onCleared()+ }+@Suppress("DEPRECATION")private fun getAppVersion(application: Application): String {val packageManager = application.packageManagerdiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.ktindex b1d7b33041..40576932d7 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt@@ -1,6 +1,7 @@package io.ente.ensuimport android.app.Application+import android.os.Buildimport io.ente.ensu.assets.AssetStoreimport io.ente.ensu.bindings.RustLogLevelimport io.ente.ensu.bindings.RustLogSink@@ -18,9 +19,11 @@ class EnsuApplication : Application() {val transcriber by lazy {val store = assetStoreval transcription = transcriptionModelAsset()- Transcriber(+ Transcriber.withCache(store.assetDir(transcription).absolutePath,store.voiceActivityModelPath().absolutePath,+ cacheDir.absolutePath,+ "${Build.FINGERPRINT}|${Build.HARDWARE}|${Build.SUPPORTED_ABIS.joinToString()}",)}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex a4c47757e0..b306bf813e 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -111,15 +111,21 @@ internal fun HomeNavigation(popExitTransition = { backExit() },) {ChatView(- onVoiceInputJob = store::trackVoiceInput,+ voiceController = store.voiceController,+ onOpenVoiceInput = { value ->+ store.openVoiceInput(+ value.text,+ value.selection.min,+ value.selection.max,+ value.composition != null,+ )+ },onOpenPacks = { navController.navigate(HomeRoute.Knowledge) },onOpenNotes = { navController.navigate(HomeRoute.NotesSetup) },hasSeenSourceIntroduction = store.hasSeenSourceIntroduction(),onCompleteSourceIntroduction = store::completeSourceIntroduction,onRetrySources = store::retrySourceInventory,chatState = appState.chat,- assetStore = store.assetStore,- llmProvider = store.llmProvider,isDrawerOpen = drawerState.currentValue == DrawerValue.Open,onMessageChange = store::updateMessageText,onSend = store::sendMessage,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.ktindex b5f9cf9162..599d384af5 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt@@ -214,15 +214,16 @@ fun HomeView(} else if (event == Lifecycle.Event.ON_STOP) {latestStore.notesStore.setForeground(false)latestStore.setChatActive(false)- latestStore.setAppForeground(false)+ if ((context as? android.app.Activity)?.isChangingConfigurations != true)+ latestStore.setAppForeground(false)}}latestStore.notesStore.setForeground(lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED))- latestStore.setAppForeground(- lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)- )+ if (lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)) {+ latestStore.setAppForeground(true)+ }lifecycleOwner.lifecycle.addObserver(observer)latestStore.setChatActive(isChatRoute &&@@ -230,12 +231,14 @@ fun HomeView()onDispose {latestStore.setChatActive(false)- latestStore.setAppForeground(false)+ if ((context as? android.app.Activity)?.isChangingConfigurations != true)+ latestStore.setAppForeground(false)lifecycleOwner.lifecycle.removeObserver(observer)}}LaunchedEffect(isChatRoute, lifecycleOwner) {+ if (!isChatRoute) latestStore.voiceController.interrupt()latestStore.setChatActive(isChatRoute &&lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.ktindex b54de38b48..9c3938b402 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt@@ -21,8 +21,6 @@ import androidx.compose.foundation.shape.RoundedCornerShapeimport androidx.compose.foundation.text.BasicTextFieldimport androidx.compose.foundation.text.KeyboardOptionsimport androidx.compose.foundation.verticalScroll-import androidx.compose.material.icons.Icons-import androidx.compose.material.icons.rounded.ErrorOutlineimport androidx.compose.material3.CircularProgressIndicatorimport androidx.compose.material3.Iconimport androidx.compose.material3.IconButton@@ -76,8 +74,9 @@ internal fun MessageInput(onAttachmentSelected: (AttachmentType) -> Unit,onRemoveAttachment: (Attachment) -> Unit,onCancelEdit: () -> Unit,- voiceInputState: VoiceInputState,- onVoiceInput: () -> Unit,+ voiceController: VoiceTranscriptionController,+ onVoiceInput: (TextFieldValue) -> Unit,+ voiceCaret: Int?,focusRequestId: Int,) {val haptic = rememberHaptics()@@ -94,12 +93,15 @@ internal fun MessageInput()}- LaunchedEffect(messageText) {- if (messageText != fieldValue.text) {+ LaunchedEffect(messageText, voiceCaret, voiceController.active) {+ if (messageText != fieldValue.text || voiceCaret != null) {fieldValue =TextFieldValue(text = messageText,- selection = TextRange(messageText.length),+ selection =+ TextRange(+ voiceCaret?.coerceIn(0, messageText.length) ?: messageText.length+ ),)}}@@ -196,50 +198,7 @@ internal fun MessageInput(}}- voiceInputState.statusText()?.let { status ->- Row(- modifier = Modifier.padding(horizontal = EnsuSpacing.sm.dp),- verticalAlignment = Alignment.CenterVertically,- ) {- when (voiceInputState) {- is VoiceInputState.Downloading,- VoiceInputState.Transcribing -> {- CircularProgressIndicator(- modifier = Modifier.size(16.dp),- strokeWidth = 2.dp,- )- }- VoiceInputState.Recording -> {- Icon(- painter = painterResource(HugeIcons.Mic02Icon),- contentDescription = null,- modifier = Modifier.size(16.dp),- tint = EnsuColor.stopButton,- )- }- is VoiceInputState.Error -> {- Icon(- imageVector = Icons.Rounded.ErrorOutline,- contentDescription = null,- modifier = Modifier.size(16.dp),- tint = EnsuColor.stopButton,- )- }- VoiceInputState.Idle -> Unit- }- Spacer(modifier = Modifier.width(EnsuSpacing.sm.dp))- Text(- text = status,- style = EnsuTypography.small,- color =- if (voiceInputState is VoiceInputState.Error) {- EnsuColor.stopButton- } else {- EnsuColor.textMuted()- },- )- }- }+ VoiceInputControls(voiceController)Box(modifier =@@ -259,7 +218,7 @@ internal fun MessageInput(modifier = Modifier.fillMaxWidth(),verticalAlignment = Alignment.CenterVertically,) {- if (supportsImageAttachments) {+ if (supportsImageAttachments && !voiceController.active) {IconButton(onClick = {haptic.perform(HapticFeedbackType.TextHandleMove)@@ -287,13 +246,14 @@ internal fun MessageInput(value = fieldValue,onValueChange = { newValue ->fieldValue = newValue- onMessageChange(newValue.text)+ if (!voiceController.active) onMessageChange(newValue.text)},modifier = Modifier.weight(1f).focusRequester(focusRequester),textStyle = EnsuTypography.message.copy(color = EnsuColor.textPrimary()),visualTransformation = LongMessageVisualTransformation,minLines = 1,- maxLines = 5,+ maxLines = if (voiceController.active) 8 else 5,+ readOnly = voiceController.active,keyboardOptions =KeyboardOptions(capitalization = KeyboardCapitalization.Sentences),cursorBrush = SolidColor(EnsuColor.accent()),@@ -314,53 +274,20 @@ internal fun MessageInput(}if (editingMessage == null) {- val isVoiceBusy =- voiceInputState is VoiceInputState.Downloading ||- voiceInputState is VoiceInputState.Transcribing- val canUseVoice =- voiceInputState.isRecording ||- (!isGenerating && !isDownloading && !isVoiceBusy)-- IconButton(- onClick = {- haptic.perform(- if (voiceInputState.isRecording) {- HapticFeedbackType.LongPress- } else {- HapticFeedbackType.TextHandleMove- }- )- onVoiceInput()+ VoiceInputButton(+ controller = voiceController,+ enabled = !isGenerating && !isDownloading && fieldValue.composition == null,+ onOpen = {+ haptic.perform(HapticFeedbackType.TextHandleMove)+ onVoiceInput(fieldValue)},- enabled = canUseVoice,- modifier = Modifier.size(36.dp),- ) {- if (isVoiceBusy) {- CircularProgressIndicator(- modifier = Modifier.size(16.dp),- strokeWidth = 2.dp,- )- } else if (voiceInputState.isRecording) {- Icon(- painter = painterResource(HugeIcons.StopIcon),- contentDescription = "Stop dictation",- modifier = Modifier.size(composerIconSize),- tint = EnsuColor.sendButton,- )- } else {- Icon(- painter = painterResource(HugeIcons.Mic02Icon),- contentDescription = "Dictate",- modifier = Modifier.size(composerIconSize),- )- }- }+ )}val canSend =sourcesReady &&(messageText.isNotBlank() || attachments.isNotEmpty()) &&- !voiceInputState.blocksSend &&+ !voiceController.active &&!hasUnsupportedImagesIconButton(diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex 89cddf3df3..6b0d17dca9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -947,6 +947,9 @@ internal class ChatStoreActions(systemPrompt = normalSystemPrompt,)+ if (normalHistorySelection.currentInputTooLarge) {+ throw ConversationException.CurrentInputTooLarge()+ }if (normalHistorySelection.wasTrimmed &&overflowBypassMessageId != userMessage.id@@ -1086,6 +1089,18 @@ internal class ChatStoreActions(throw err} catch (_: LlmException.Cancelled) {interrupted = true+ } catch (_: ConversationException.CurrentInputTooLarge) {+ if (isActive() && !stopRequested) {+ state.update {+ it.copy(+ chat =+ it.chat.copy(+ transientAssistantError = CURRENT_INPUT_TOO_LARGE_MESSAGE,+ transientAssistantParentId = userMessage.id,+ )+ )+ }+ }} catch (err: ModelMemoryUnavailable) {if (isActive() && !stopRequested) {state.update {@@ -1326,9 +1341,12 @@ internal class ChatStoreActions(chat =it.chat.copy(transientAssistantError =- (error as? ConversationException.Other)?.detail- ?: error.message- ?: "Unable to prepare conversation",+ if (error is ConversationException.CurrentInputTooLarge)+ CURRENT_INPUT_TOO_LARGE_MESSAGE+ else+ (error as? ConversationException.Other)?.detail+ ?: error.message+ ?: "Unable to prepare conversation",transientAssistantParentId = userMessage.id,))@@ -1843,6 +1861,7 @@ internal class ChatStoreActions(val inputTokens: Int,val inputBudget: Int,val wasTrimmed: Boolean,+ val currentInputTooLarge: Boolean,)private data class GenerationLimits(@@ -1914,6 +1933,7 @@ internal class ChatStoreActions(inputTokens,inputBudget,inputTokens > inputBudget,+ systemTokens + promptTokens > inputBudget,)}@@ -1944,7 +1964,13 @@ internal class ChatStoreActions()}- return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)+ return HistorySelection(+ selected,+ inputTokens,+ inputBudget,+ inputTokens > inputBudget,+ false,+ )}private fun resolveGenerationLimits(contextLength: Int): GenerationLimits {@@ -2091,6 +2117,9 @@ internal class ChatStoreActions(}companion object {+ private const val CURRENT_INPUT_TOO_LARGE_MESSAGE =+ "This message is too long for the model, even without earlier messages. " ++ "Shorten it or split it into smaller messages. Your full text is still available to edit or copy."private const val MEDIA_MARKER = "<__media__>"private const val OVERFLOW_SAFETY_TOKENS = 256private const val IMAGE_TOKEN_ESTIMATE = 768diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex 31f9beb63b..9bce45e391 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -1,9 +1,5 @@package io.ente.ensu.chat-import android.Manifest-import android.content.pm.PackageManager-import androidx.activity.compose.rememberLauncherForActivityResult-import androidx.activity.result.contract.ActivityResultContractsimport androidx.compose.animation.AnimatedContentimport androidx.compose.animation.core.FastOutSlowInEasingimport androidx.compose.animation.core.tween@@ -36,28 +32,22 @@ import androidx.compose.runtime.keyimport androidx.compose.runtime.mutableIntStateOfimport androidx.compose.runtime.mutableStateOfimport androidx.compose.runtime.remember-import androidx.compose.runtime.rememberUpdatedStateimport androidx.compose.runtime.setValueimport androidx.compose.ui.Alignmentimport androidx.compose.ui.Modifierimport androidx.compose.ui.layout.onGloballyPositioned-import androidx.compose.ui.platform.LocalContextimport androidx.compose.ui.platform.LocalDensityimport androidx.compose.ui.platform.LocalFocusManagerimport androidx.compose.ui.unit.dp-import androidx.core.content.ContextCompat-import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.designsystem.EnsuColorimport io.ente.ensu.designsystem.EnsuSpacingimport io.ente.ensu.device.ChatDeviceCapability-import io.ente.ensu.llm.LlmProvider-import kotlinx.coroutines.Job@Composable-fun ChatView(+internal fun ChatView(chatState: ChatState,- assetStore: AssetStore,- llmProvider: LlmProvider,+ voiceController: VoiceTranscriptionController,+ onOpenVoiceInput: (androidx.compose.ui.text.input.TextFieldValue) -> Unit,isDrawerOpen: Boolean,onMessageChange: (String) -> Unit,onSend: () -> Unit,@@ -74,7 +64,6 @@ fun ChatView(onDismissUnsupportedDeviceDialog: () -> Unit,onOverflowTrim: () -> Unit,onOverflowCancel: () -> Unit,- onVoiceInputJob: (Job) -> Unit,onOpenPacks: () -> Unit,onOpenNotes: () -> Unit,hasSeenSourceIntroduction: Boolean,@@ -82,28 +71,13 @@ fun ChatView(onRetrySources: () -> Unit,) {val density = LocalDensity.current- val context = LocalContext.currentvar inputBarHeightDp by remember { mutableStateOf(0.dp) }- val latestMessageText by rememberUpdatedState(chatState.messageText)- val latestOnMessageChange by rememberUpdatedState(onMessageChange)val sessionKey = chatState.currentSessionId ?: "new-session"var introductionSeen byremember(hasSeenSourceIntroduction) { mutableStateOf(hasSeenSourceIntroduction) }- val latestSessionKey by rememberUpdatedState(sessionKey)val unsupportedCapability =chatState.deviceCapability as? ChatDeviceCapability.UnsupportedLowMemoryval isChatUnsupported = unsupportedCapability != null- var pendingVoiceSessionKey by remember { mutableStateOf<String?>(null) }- val voiceController =- rememberVoiceTranscriptionController(- assetStore = assetStore,- llmProvider = llmProvider,- onVoiceInputJob = onVoiceInputJob,- onTranscript = { transcript ->- latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))- },- )-val editingMessage byremember(chatState.editingMessageId, chatState.messages) {derivedStateOf {@@ -112,30 +86,6 @@ fun ChatView(}}}- val canStartVoiceInput =- !chatState.isGenerating &&- !chatState.isDownloading &&- chatState.isModelDownloaded &&- !isChatUnsupported &&- editingMessage == null- val latestCanStartVoiceInput by rememberUpdatedState(canStartVoiceInput)-- val microphonePermissionLauncher =- rememberLauncherForActivityResult(contract = ActivityResultContracts.RequestPermission()) {- granted ->- val requestedSessionKey = pendingVoiceSessionKey- pendingVoiceSessionKey = null- if (granted) {- if (requestedSessionKey != null) {- voiceController.startRecording {- latestSessionKey == requestedSessionKey && latestCanStartVoiceInput- }- }- } else {- voiceController.onPermissionDenied()- }- }-val showDownloadOnboarding byremember(chatState.isModelDownloaded,@@ -202,11 +152,6 @@ fun ChatView(}}- LaunchedEffect(sessionKey) {- pendingVoiceSessionKey = null- voiceController.cancelActiveVoiceInput()- }-if (showIntroduction) {SourceIntroductionSheet(onDismiss = {@@ -290,7 +235,8 @@ fun ChatView(inputBarHeightDp = with(density) { coords.size.height.toDp() }})- } else if (chatState.isModelStateKnown && !showDownloadOnboarding) {+ } else if (chatState.isModelStateKnown) {+ val voicePreview = voiceController.composerPreview(chatState.messageText)MessageInput(modifier =Modifier.fillMaxWidth()@@ -299,7 +245,7 @@ fun ChatView(.onGloballyPositioned { coords ->inputBarHeightDp = with(density) { coords.size.height.toDp() }},- messageText = chatState.messageText,+ messageText = voicePreview?.text ?: chatState.messageText,attachments = chatState.attachments,supportsImageAttachments = chatState.supportsImageAttachments,editingMessage = editingMessage,@@ -318,30 +264,12 @@ fun ChatView(onAttachmentSelected = onAttachmentSelected,onRemoveAttachment = onRemoveAttachment,onCancelEdit = onCancelEdit,- voiceInputState = voiceController.state,- onVoiceInput = {- if (voiceController.state.isRecording) {- voiceController.stopAndTranscribe()- } else {- val hasMicrophonePermission =- ContextCompat.checkSelfPermission(- context,- Manifest.permission.RECORD_AUDIO,- ) == PackageManager.PERMISSION_GRANTED- if (canStartVoiceInput && hasMicrophonePermission) {- val requestedSessionKey = sessionKey- voiceController.startRecording {- latestSessionKey == requestedSessionKey &&- latestCanStartVoiceInput- }- } else if (canStartVoiceInput) {- pendingVoiceSessionKey = sessionKey- microphonePermissionLauncher.launch(- Manifest.permission.RECORD_AUDIO- )- }- }+ voiceController = voiceController,+ onVoiceInput = { value ->+ onOpenVoiceInput(value)+ focusManager.clearFocus()},+ voiceCaret = voicePreview?.caretUtf16?.toInt() ?: voiceController.composerCaret,focusRequestId = focusRequestId,)}@@ -398,15 +326,3 @@ fun ChatView(}}}--private fun appendVoiceTranscript(currentText: String, transcript: String): String {- val cleaned = transcript.trim()- if (cleaned.isBlank()) {- return currentText- }- val trimmedDraft = currentText.trimEnd()- if (trimmedDraft.isBlank()) {- return cleaned- }- return "$trimmedDraft $cleaned"-}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex 1aa2754e29..52b58d1407 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -1,51 +1,55 @@package io.ente.ensu.chatimport android.annotation.SuppressLint+import android.content.Context+import android.content.pm.ApplicationInfoimport android.media.AudioFormat+import android.media.AudioManagerimport android.media.AudioRecordimport android.media.MediaRecorder-import android.util.Log-import androidx.compose.runtime.Composable-import androidx.compose.runtime.DisposableEffectimport androidx.compose.runtime.getValueimport androidx.compose.runtime.mutableStateOf-import androidx.compose.runtime.rememberimport androidx.compose.runtime.setValue-import androidx.lifecycle.Lifecycle-import androidx.lifecycle.LifecycleEventObserver-import androidx.lifecycle.compose.LocalLifecycleOwnerimport io.ente.ensu.assets.AssetStore-import io.ente.ensu.bindings.AssetDownloadException-import io.ente.ensu.bindings.LlmException-import io.ente.ensu.bindings.TranscriptionException+import io.ente.ensu.bindings.DictationException+import io.ente.ensu.bindings.DictationInsertion+import io.ente.ensu.bindings.DictationPhase+import io.ente.ensu.bindings.DictationSession+import io.ente.ensu.bindings.DictationSnapshot+import io.ente.ensu.bindings.DictationStopReason+import io.ente.ensu.bindings.DictationStream+import io.ente.ensu.bindings.DictationStreamCallbackimport io.ente.ensu.bindings.transcriptionModelAssetimport io.ente.ensu.bindings.voiceActivityModelAsset+import io.ente.ensu.device.AndroidDeviceCapabilityProvider+import io.ente.ensu.device.supportsLiveVoiceimport io.ente.ensu.llm.LlmProvider-import io.ente.ensu.llm.ModelMemoryUnavailable-import java.io.ByteArrayOutputStream+import io.ente.ensu.logging.FileLogRepository+import io.ente.ensu.logging.LogLevelimport java.io.IOException-import java.util.UUID-import kotlin.coroutines.CoroutineContext-import kotlin.coroutines.EmptyCoroutineContext-import kotlin.coroutines.coroutineContext+import java.nio.ByteBuffer+import java.nio.ByteOrder+import java.util.concurrent.atomic.AtomicBoolean+import kotlin.math.log10+import kotlin.math.sqrtimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.CoroutineScope-import kotlinx.coroutines.CoroutineStartimport kotlinx.coroutines.Dispatchersimport kotlinx.coroutines.Jobimport kotlinx.coroutines.NonCancellableimport kotlinx.coroutines.SupervisorJob+import kotlinx.coroutines.asyncimport kotlinx.coroutines.cancel-import kotlinx.coroutines.cancelAndJoin+import kotlinx.coroutines.coroutineScopeimport kotlinx.coroutines.delay-import kotlinx.coroutines.ensureActive-import kotlinx.coroutines.isActiveimport kotlinx.coroutines.launchimport kotlinx.coroutines.withContextinternal sealed interface VoiceInputState {data object Idle : VoiceInputState+ data object Preparing : VoiceInputState+data object Recording : VoiceInputStatedata class Downloading(val percent: Int?) : VoiceInputState@@ -58,440 +62,464 @@ internal sealed interface VoiceInputState {internal val VoiceInputState.isRecording: Booleanget() = this is VoiceInputState.Recording-internal val VoiceInputState.isWorking: Boolean- get() =- this is VoiceInputState.Recording ||- this is VoiceInputState.Downloading ||- this is VoiceInputState.Transcribing--internal val VoiceInputState.blocksSend: Boolean- get() = this is VoiceInputState.Recording || this is VoiceInputState.Transcribing--internal fun VoiceInputState.statusText(): String? =- when (this) {- VoiceInputState.Idle -> null- VoiceInputState.Recording -> "Listening..."- is VoiceInputState.Downloading -> {- val suffix = percent?.let { " ($it%)" }.orEmpty()- "Downloading voice model...$suffix"- }- VoiceInputState.Transcribing -> null- is VoiceInputState.Error -> message- }--@Composable-internal fun rememberVoiceTranscriptionController(- assetStore: AssetStore,- llmProvider: LlmProvider,- onTranscript: (String) -> Unit,- onVoiceInputJob: (Job) -> Unit,-): VoiceTranscriptionController {- val lifecycleOwner = LocalLifecycleOwner.current- val controller =- remember(llmProvider) {- VoiceTranscriptionController(- assetStore,- llmProvider,- onTranscript,- onVoiceInputJob,- )- }- DisposableEffect(controller, lifecycleOwner) {- val observer = LifecycleEventObserver { _, event ->- if (event == Lifecycle.Event.ON_PAUSE) {- controller.cancelActiveVoiceInput()- }- }- lifecycleOwner.lifecycle.addObserver(observer)- onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }- }- DisposableEffect(controller) { onDispose { controller.dispose() } }- return controller-}-internal class VoiceTranscriptionController(+ context: Context,private val assetStore: AssetStore,private val llmProvider: LlmProvider,- private val onTranscript: (String) -> Unit,+ private val logRepository: FileLogRepository,private val onVoiceInputJob: (Job) -> Unit,+ private val onInsert: (DictationInsertion, String) -> Boolean,) {+ private val diagnosticsEnabled =+ context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0++ private fun diagnostic(event: String) {+ if (diagnosticsEnabled) {+ logRepository.log(LogLevel.Info, event, tag = "EnsuVoice")+ }+ }++ private val audioManager = context.getSystemService(AudioManager::class.java)+ private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)+ private val usesLiveDictation =+ supportsLiveVoice(+ AndroidDeviceCapabilityProvider(context).chatCapability().totalMemoryBytes+ )private val modelAssets =listOf(transcriptionModelAsset(),voiceActivityModelAsset(),)+ private var session: DictationSession? = null+ private var stream: DictationStream? = null+ private var capture: LiveMicrophone? = null+ private var work: Job? = null+ private var finishingWatchdog: Job? = null+ private var attemptFinished = false+ private var insertion: DictationInsertion? = null+ private var foreground = true+ private var generation = 0L+ val sessionVersion: Long+ get() = generation++ private var initialRecordPending = false+ private var awaitingPermission = false+ private var startOnForeground = false++ var active by mutableStateOf(false)+ private set- private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)- private val bufferLock = Any()+ var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)+ private set- private var audioRecord: AudioRecord? = null- private var preparingRecordingJob: Job? = null- private var recordingJob: Job? = null- private var transcriptionPreloadJob: Job? = null- private var voiceOwner = UUID.randomUUID().toString()- private var transientErrorJob: Job? = null- private var recordedPcm = ByteArrayOutputStream()- private var recordingSampleRate = preferredSampleRate+ var snapshot: DictationSnapshot? by mutableStateOf(null)+ private set- var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)+ private var notice: String? = null++ var audioLevels: List<Float> by mutableStateOf(List(24) { 0f })private set- private fun launchVoiceWork(- context: CoroutineContext = EmptyCoroutineContext,- block: suspend CoroutineScope.() -> Unit,- ): Job =- scope.launch(context, start = CoroutineStart.LAZY, block = block).also {- onVoiceInputJob(it)- it.start()- }+ var composerCaret: Int? by mutableStateOf(null)++ var hasActiveWork by mutableStateOf(false)+ private set- fun onPermissionDenied() {- state = VoiceInputState.Error("Microphone permission is required for voice input.")+ val canOpen: Boolean+ get() = !active && !hasActiveWork++ fun open(insertion: DictationInsertion) {+ if (!canOpen) return+ generation+++ initialRecordPending = true+ attemptFinished = false+ this.insertion = insertion+ session = DictationSession(live = usesLiveDictation)+ snapshot = session?.snapshot()+ notice = null+ audioLevels = List(24) { 0f }+ active = true+ state = VoiceInputState.Idle}- @SuppressLint("MissingPermission")- fun startRecording(canStartRecording: () -> Boolean = { true }) {- transientErrorJob?.cancel()- if (state.isWorking || preparingRecordingJob?.isActive == true) {- return- }+ fun composerPreview(original: String): io.ente.ensu.bindings.DictationInsertionResult? {+ val value = snapshot ?: return null+ if (!active) return null+ return runCatching { insertion?.preview(original, value.text + value.hypothesis) }+ .getOrNull()+ }- preparingRecordingJob = launchVoiceWork {- try {- ensureTranscriptionModelDownloaded()- if (!isActive || !canStartRecording()) {- if (state is VoiceInputState.Downloading) {- state = VoiceInputState.Idle- }- return@launchVoiceWork- }- beginRecording()- preloadTranscriptionModel()- } catch (error: CancellationException) {- throw error- } catch (error: UnsatisfiedLinkError) {- Log.w(TAG, "Voice transcription is not available on this device", error)- state =- VoiceInputState.Error("Voice transcription is not available on this device.")- } catch (error: TranscriptionException) {- Log.w(TAG, "Voice model preparation failed: ${error.message}", error)- state = VoiceInputState.Error(transcriptionErrorMessage(error))- } catch (_: AssetDownloadException.Cancelled) {- state = VoiceInputState.Idle- } catch (error: AssetDownloadException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (_: LlmException.Cancelled) {- state = VoiceInputState.Idle- } catch (error: LlmException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (error: Throwable) {- Log.w(- TAG,- "Voice model preparation failed: ${error.javaClass.simpleName}: ${error.message}",- error,- )- state = VoiceInputState.Error("Could not transcribe voice input.")- }- }+ fun takeInitialRecord(): Boolean = initialRecordPending.also { initialRecordPending = false }++ fun permissionRequested() {+ awaitingPermission = true}- @SuppressLint("MissingPermission")- private fun beginRecording() {- if (state is VoiceInputState.Recording || state is VoiceInputState.Transcribing) {- return+ fun permissionResult(granted: Boolean) {+ awaitingPermission = false+ if (granted) {+ if (foreground) record() else startOnForeground = true+ } else {+ discard()+ state =+ VoiceInputState.Error("Microphone permission is required. Enable it in Settings.")}+ }- synchronized(bufferLock) { recordedPcm = ByteArrayOutputStream() }- recordingSampleRate = preferredSampleRate- state = VoiceInputState.Recording+ fun setForeground(active: Boolean) {+ foreground = active+ if (active) {+ if (startOnForeground) {+ startOnForeground = false+ record()+ } else session?.let { refresh(it, generation) }+ } else if (!awaitingPermission && !startOnForeground) interrupt()+ }- recordingJob =- launchVoiceWork(Dispatchers.IO) {- var recorder: AudioRecord? = null- try {- val activeRecorder = createRecorder(recordingSampleRate)- recorder = activeRecorder- audioRecord = activeRecorder- recordingSampleRate =- activeRecorder.sampleRate.takeIf { it > 0 } ?: preferredSampleRate- activeRecorder.startRecording()- val buffer = ByteArray(readBufferSize(recordingSampleRate))- while (coroutineContext.isActive) {- val read = activeRecorder.read(buffer, 0, buffer.size)- when {- read > 0 ->- synchronized(bufferLock) { recordedPcm.write(buffer, 0, read) }- read < 0 -> throw IOException("Audio recorder failed with code $read")+ fun record() {+ val owner = session ?: return+ if (!active || !foreground || hasActiveWork || attemptFinished) return+ notice = null+ state = VoiceInputState.Preparing+ val token = generation+ hasActiveWork = true+ work =+ scope+ .launch {+ var ownedCapture: LiveMicrophone? = null+ var ownedStream: DictationStream? = null+ try {+ if (+ !withContext(Dispatchers.IO) {+ modelAssets.all { assetStore.isDownloaded(it) }+ }+ ) {+ state = VoiceInputState.Downloading(null)+ assetStore.download(modelAssets) { progress ->+ scope.launch {+ if (token == generation && state is VoiceInputState.Downloading)+ state =+ VoiceInputState.Downloading(progress.percentage.toInt())+ }+ }}- }- } catch (error: Throwable) {- if (error is CancellationException) {+ if (token != generation || !foreground) return@launch+ state = VoiceInputState.Preparing+ val activeStream = DictationStream(owner, 16000u, 1u)+ stream = activeStream+ ownedStream = activeStream+ val microphone =+ LiveMicrophone(+ activeStream,+ audioManager,+ onLevel = { level ->+ scope.launch {+ if (token == generation && state.isRecording)+ audioLevels = audioLevels.drop(1) + level+ }+ },+ ) {+ scope.launch {+ if (token == generation)+ interrupt(+ "Microphone interrupted. Review the text before continuing."+ )+ }+ }+ capture = microphone+ ownedCapture = microphone+ coroutineScope {+ val hardwareReady = async(Dispatchers.IO) { microphone.prepare() }+ val callback =+ object : DictationStreamCallback {+ override fun onReady() {+ scope.launch {+ try {+ hardwareReady.await()+ val value = owner.snapshot()+ if (+ token != generation ||+ !foreground ||+ !active ||+ stream !== activeStream ||+ !value.workerActive ||+ value.phase != DictationPhase.LISTENING+ )+ return@launch+ microphone.start(scope)+ state = VoiceInputState.Recording+ audioLevels = List(24) { 0f }+ notice = null+ refresh(owner, token)+ } catch (_: Exception) {+ if (token == generation)+ interrupt(+ "Could not start the microphone. Tap the microphone to start again."+ )+ }+ }+ }++ override fun onChange() {+ scope.launch { refresh(owner, token) }+ }++ override fun onCaptureClosed() {+ microphone.stop()+ scope.launch {+ watchFinalization(owner, activeStream, token)+ }+ }+ }+ llmProvider.runLiveVoice(activeStream, callback)+ }+ } catch (error: CancellationException) {throw error- }- Log.w(TAG, "Could not record microphone audio", error)- withContext(Dispatchers.Main.immediate) {- releaseVoice()- if (state is VoiceInputState.Recording) {- state = VoiceInputState.Error("Could not record microphone audio.")+ } catch (error: Exception) {+ ownedStream?.let { runCatching { it.abandon(DictationStopReason.FAILURE) } }+ diagnostic(+ if (error is DictationException.InvalidOperation)+ "failure ${error.detail}"+ else "failure ${error.javaClass.simpleName}"+ )+ if (token == generation && notice == null) {+ notice = "Voice input stopped. Review the available text or try again."}- }- } finally {- runCatching {- if (recorder?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {- recorder.stop()+ } finally {+ withContext(NonCancellable) { ownedCapture?.stopAndDrain() }+ hasActiveWork = false+ if (token == generation) {+ finishingWatchdog?.cancel()+ finishingWatchdog = null+ capture = null+ stream = null+ attemptFinished = true+ refresh(owner, token)}}- recorder?.release()- if (audioRecord === recorder) {- audioRecord = null- }}- }+ .also(onVoiceInputJob)}- fun stopAndTranscribe() {- if (state !is VoiceInputState.Recording) {+ private fun refresh(owner: DictationSession, token: Long) {+ if (token != generation || !active) return+ val value = owner.snapshot()+ snapshot = value+ if (!attemptFinished) {+ if (value.phase == DictationPhase.FINISHING) state = VoiceInputState.Transcribingreturn}-- state = VoiceInputState.Transcribing- preparingRecordingJob?.cancel()- preparingRecordingJob = null- launchVoiceWork {- recordingJob?.cancelAndJoin()- recordingJob = null-- val pcm = synchronized(bufferLock) { recordedPcm.toByteArray() }- if (pcm.size < minimumRecordingBytes(recordingSampleRate)) {- releaseVoice()- showTransientError("No speech captured.")- return@launchVoiceWork+ if (!foreground) return+ val text = value.text + value.hypothesis+ val added = text.isNotBlank() && insertion?.let { onInsert(it, text) } == true+ val completedState =+ when {+ added && !value.incomplete -> VoiceInputState.Idle+ text.isBlank() -> VoiceInputState.Error(notice ?: "No speech detected.")+ else -> VoiceInputState.Error("Voice input stopped.")}-- transcribeRecording(pcm, recordingSampleRate)- }+ active = false+ snapshot = null+ session = null+ insertion = null+ audioLevels = List(24) { 0f }+ state = completedState}- fun cancelActiveVoiceInput() {- releaseVoice()- val wasPreparingRecording = preparingRecordingJob?.isActive == true- preparingRecordingJob?.cancel()- preparingRecordingJob = null-- if (state !is VoiceInputState.Recording) {- if (wasPreparingRecording && state is VoiceInputState.Downloading) {- state = VoiceInputState.Idle- }+ private fun watchFinalization(+ owner: DictationSession,+ finishingStream: DictationStream,+ token: Long,+ ) {+ if (+ token != generation ||+ !active ||+ stream !== finishingStream ||+ !owner.snapshot().workerActive ||+ finishingWatchdog != null+ )return- }-- state = VoiceInputState.Idle- recordingJob?.cancel()- recordingJob = null- stopActiveRecorder()- synchronized(bufferLock) { recordedPcm.reset() }- }-- fun dispose() {- releaseVoice()- transientErrorJob?.cancel()- preparingRecordingJob?.cancel()- scope.cancel()- runCatching { audioRecord?.release() }- audioRecord = null- }-- private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) {- val owner = voiceOwner- try {- ensureTranscriptionModelDownloaded()- transcriptionPreloadJob?.join()- coroutineContext.ensureActive()- state = VoiceInputState.Transcribing- val transcript = llmProvider.transcribe(sampleRate.toUInt(), pcm).trim()-- if (transcript.isBlank()) {- showTransientError("No speech detected.")- } else {- onTranscript(transcript)- state = VoiceInputState.Idle- }- } catch (error: CancellationException) {- throw error- } catch (error: UnsatisfiedLinkError) {- Log.w(TAG, "Voice transcription is not available on this device", error)- state = VoiceInputState.Error("Voice transcription is not available on this device.")- } catch (error: TranscriptionException) {- Log.w(TAG, "Voice transcription failed: ${error.message}", error)- state = VoiceInputState.Error(transcriptionErrorMessage(error))- } catch (error: ModelMemoryUnavailable) {- state = VoiceInputState.Error(checkNotNull(error.message))- } catch (_: AssetDownloadException.Cancelled) {- state = VoiceInputState.Idle- } catch (error: AssetDownloadException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (_: LlmException.Cancelled) {- state = VoiceInputState.Idle- } catch (error: LlmException) {- Log.w(TAG, "Voice model download failed: ${error.message}", error)- state = VoiceInputState.Error(downloadErrorMessage)- } catch (error: Throwable) {- Log.w(- TAG,- "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",- error,+ finishingWatchdog = scope.launch {+ delay(10_000)+ if (+ token != generation ||+ !active ||+ stream !== finishingStream ||+ !owner.snapshot().workerActive)- state = VoiceInputState.Error("Could not transcribe voice input.")- } finally {- withContext(NonCancellable) { llmProvider.releaseVoice(owner) }+ return@launch+ capture?.stop()+ runCatching { finishingStream.abandon(DictationStopReason.FAILURE) }+ work?.cancel()+ snapshot = owner.snapshot()+ notice =+ "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."}}- private fun preloadTranscriptionModel() {- voiceOwner = UUID.randomUUID().toString()- val owner = voiceOwner- transcriptionPreloadJob = launchVoiceWork {- try {- llmProvider.prewarmVoice(owner)- } catch (error: CancellationException) {- throw error- } catch (error: Throwable) {- Log.d(TAG, "Voice preload skipped", error)- }+ fun stop() {+ val reason = DictationStopReason.DONE+ runCatching { session?.stop(reason) }+ snapshot = session?.snapshot()+ if (snapshot?.workerActive != true) return+ state = VoiceInputState.Transcribing+ audioLevels = List(24) { 0f }+ val active = stream+ val microphone = capture+ if (active != null) session?.let { watchFinalization(it, active, generation) }+ scope.launch {+ microphone?.stopAndDrain()+ runCatching { active?.stop(reason) }}}- private fun releaseVoice() {- transcriptionPreloadJob?.cancel()- transcriptionPreloadJob = null- val owner = voiceOwner- voiceOwner = UUID.randomUUID().toString()- launchVoiceWork(NonCancellable) { llmProvider.releaseVoice(owner) }- }-- private suspend fun ensureTranscriptionModelDownloaded() {- val activeJob = coroutineContext[Job]-- if (withContext(Dispatchers.IO) { modelAssets.all { assetStore.isDownloaded(it) } }) {+ fun interrupt(message: String = "Voice input stopped.") {+ if (!active || attemptFinished) return+ if (!hasActiveWork && stream == null) {+ discard()return}-- setStateIfJobActive(activeJob, VoiceInputState.Downloading(null))- assetStore.download(modelAssets) { progress ->- postStateIfJobActive(- activeJob,- VoiceInputState.Downloading(progress.percentage.toInt().coerceIn(0, 100)),- )- }+ capture?.stop()+ runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }+ work?.cancel()+ notice = message+ state = VoiceInputState.Transcribing}- private suspend fun setStateIfJobActive(activeJob: Job?, nextState: VoiceInputState) {- if (activeJob?.isActive == false) {- return- }- withContext(Dispatchers.Main.immediate) {- if (activeJob?.isActive != false) {- state = nextState- }- }+ fun discard() {+ awaitingPermission = false+ startOnForeground = false+ generation+++ finishingWatchdog?.cancel()+ finishingWatchdog = null+ capture?.stop()+ runCatching { stream?.abandon(DictationStopReason.DISCARD) }+ work?.cancel()+ active = false+ state = VoiceInputState.Idle+ snapshot = null+ session = null+ insertion = null+ stream = null+ capture = null}- private fun postStateIfJobActive(activeJob: Job?, nextState: VoiceInputState) {- if (activeJob?.isActive == false) {- return- }- scope.launch(Dispatchers.Main.immediate) {- if (activeJob?.isActive != false) {- state = nextState- }- }+ fun dispose() {+ discard()+ scope.cancel()}+}- private fun stopActiveRecorder() {- val recorder = audioRecord ?: return- runCatching {- if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) {- recorder.stop()- }- }- }+private class LiveMicrophone(+ private val stream: DictationStream,+ private val audioManager: AudioManager,+ private val onLevel: (Float) -> Unit,+ private val onInterrupted: () -> Unit,+) {+ private val stopped = AtomicBoolean(false)+ @Volatile private var recorder: AudioRecord? = null+ private var readJob: Job? = null@SuppressLint("MissingPermission")- private fun createRecorder(sampleRate: Int): AudioRecord {- val minBufferSize =+ fun prepare() {+ if (stopped.get() || recorder != null) return+ val minimum =AudioRecord.getMinBufferSize(- sampleRate,+ 16000,AudioFormat.CHANNEL_IN_MONO,AudioFormat.ENCODING_PCM_16BIT,)- if (minBufferSize <= 0) {- throw IOException("Audio recorder buffer unavailable")+ if (minimum <= 0) throw IOException("Microphone format unavailable")+ val active =+ AudioRecord.Builder()+ .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)+ .setAudioFormat(+ AudioFormat.Builder()+ .setSampleRate(16000)+ .setChannelMask(AudioFormat.CHANNEL_IN_MONO)+ .setEncoding(AudioFormat.ENCODING_PCM_16BIT)+ .build()+ )+ .setBufferSizeInBytes(maxOf(minimum * 2, 6400))+ .build()+ if (active.state != AudioRecord.STATE_INITIALIZED) {+ active.release()+ throw IOException("Microphone unavailable")}+ recorder = active+ }- return AudioRecord.Builder()- .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)- .setAudioFormat(- AudioFormat.Builder()- .setEncoding(AudioFormat.ENCODING_PCM_16BIT)- .setSampleRate(sampleRate)- .setChannelMask(AudioFormat.CHANNEL_IN_MONO)- .build()- )- .setBufferSizeInBytes(minBufferSize * 2)- .build()- .also { recorder ->- if (recorder.state != AudioRecord.STATE_INITIALIZED) {- recorder.release()- throw IOException("Audio recorder failed to initialize")+ @SuppressLint("MissingPermission")+ fun start(scope: CoroutineScope) {+ if (stopped.get()) return+ val active = recorder ?: return+ try {+ active.startRecording()+ if (active.recordingState != AudioRecord.RECORDSTATE_RECORDING)+ throw IOException("Microphone did not start")+ } catch (error: Exception) {+ active.release()+ recorder = null+ throw error+ }+ var routedDevice = active.routedDevice?.id+ readJob =+ scope.launch(Dispatchers.IO) {+ val buffer = ByteBuffer.allocateDirect(3200).order(ByteOrder.LITTLE_ENDIAN)+ try {+ while (!stopped.get()) {+ buffer.clear()+ val read = active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)+ if (stopped.get() && read <= 0) break+ if (+ read <= 0 ||+ audioManager.isMicrophoneMute ||+ (android.os.Build.VERSION.SDK_INT >= 29 &&+ active.activeRecordingConfiguration?.isClientSilenced ==+ true) ||+ (routedDevice != null && active.routedDevice?.id != routedDevice)+ ) {+ throw IOException("Microphone interrupted")+ }+ routedDevice = active.routedDevice?.id+ buffer.position(0)+ buffer.limit(read)+ var energy = 0.0+ for (index in 0 until read / 2) {+ val sample = buffer.getShort(index * 2).toDouble() / 32768.0+ energy += sample * sample+ }+ val rms = sqrt(energy / (read / 2).coerceAtLeast(1))+ onLevel(+ ((20 * log10(rms.coerceAtLeast(0.00001)) + 55) / 45)+ .toFloat()+ .coerceIn(0f, 1f)+ )+ stream.feed(buffer)+ }+ } catch (_: Exception) {+ if (!stopped.get()) {+ runCatching { stream.abandon(DictationStopReason.FAILURE) }+ onInterrupted()+ }+ } finally {+ stop()+ active.release()+ recorder = null}}}- private fun readBufferSize(sampleRate: Int): Int {- val minBufferSize =- AudioRecord.getMinBufferSize(- sampleRate,- AudioFormat.CHANNEL_IN_MONO,- AudioFormat.ENCODING_PCM_16BIT,- )- return maxOf(minBufferSize, 4096)- }-- private fun minimumRecordingBytes(sampleRate: Int): Int = sampleRate / 4 * bytesPerSample-- private val downloadErrorMessage =- "Voice model download failed. Check your connection and try again."-- private fun transcriptionErrorMessage(error: TranscriptionException): String =- when (error) {- is TranscriptionException.NotDownloaded,- is TranscriptionException.VadNotDownloaded,- is TranscriptionException.Transcribe -> "Voice model could not be loaded."- is TranscriptionException.InvalidAudio -> "Could not detect speech in this recording."- is TranscriptionException.StorageFull ->- "Not enough storage space. Please free up space and try again."- is TranscriptionException.Io -> "Could not transcribe voice input."- }-- private fun showTransientError(message: String) {- val errorState = VoiceInputState.Error(message)- transientErrorJob?.cancel()- state = errorState- transientErrorJob = scope.launch {- delay(transientErrorMillis)- if (state == errorState) {- state = VoiceInputState.Idle- }+ suspend fun stopAndDrain() {+ stop()+ readJob?.join()+ if (readJob == null) {+ recorder?.release()+ recorder = null}}- private companion object {- const val TAG = "VoiceInput"- const val preferredSampleRate = 16_000- const val bytesPerSample = 2- const val transientErrorMillis = 10_000L+ fun stop() {+ if (stopped.compareAndSet(false, true)) runCatching { recorder?.stop() }}}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.ktnew file mode 100644index 0000000000..b85adeef5c--- /dev/null+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt@@ -0,0 +1,212 @@+package io.ente.ensu.chat++import android.Manifest+import android.content.Intent+import android.content.pm.PackageManager+import android.net.Uri+import android.provider.Settings+import androidx.activity.compose.rememberLauncherForActivityResult+import androidx.activity.result.contract.ActivityResultContracts+import androidx.compose.animation.core.animateFloatAsState+import androidx.compose.animation.core.tween+import androidx.compose.foundation.Canvas+import androidx.compose.foundation.background+import androidx.compose.foundation.clickable+import androidx.compose.foundation.layout.Arrangement+import androidx.compose.foundation.layout.Box+import androidx.compose.foundation.layout.Row+import androidx.compose.foundation.layout.Spacer+import androidx.compose.foundation.layout.height+import androidx.compose.foundation.layout.padding+import androidx.compose.foundation.layout.size+import androidx.compose.foundation.shape.CircleShape+import androidx.compose.material.icons.Icons+import androidx.compose.material.icons.rounded.ArrowDownward+import androidx.compose.material3.CircularProgressIndicator+import androidx.compose.material3.Icon+import androidx.compose.material3.IconButton+import androidx.compose.material3.Text+import androidx.compose.runtime.Composable+import androidx.compose.runtime.LaunchedEffect+import androidx.compose.runtime.getValue+import androidx.compose.runtime.mutableStateOf+import androidx.compose.runtime.saveable.rememberSaveable+import androidx.compose.runtime.setValue+import androidx.compose.ui.Alignment+import androidx.compose.ui.Modifier+import androidx.compose.ui.geometry.CornerRadius+import androidx.compose.ui.geometry.Offset+import androidx.compose.ui.geometry.Size+import androidx.compose.ui.platform.LocalContext+import androidx.compose.ui.res.painterResource+import androidx.compose.ui.semantics.LiveRegionMode+import androidx.compose.ui.semantics.contentDescription+import androidx.compose.ui.semantics.liveRegion+import androidx.compose.ui.semantics.semantics+import androidx.compose.ui.unit.dp+import androidx.core.content.ContextCompat+import io.ente.ensu.bindings.DictationPhase+import io.ente.ensu.designsystem.EnsuColor+import io.ente.ensu.designsystem.EnsuSpacing+import io.ente.ensu.designsystem.EnsuTypography+import io.ente.ensu.designsystem.HugeIcons++@Composable+internal fun VoiceInputControls(controller: VoiceTranscriptionController) {+ val state = controller.state+ if (!controller.active && state !is VoiceInputState.Error) return+ val permissionNeeded =+ (state as? VoiceInputState.Error)?.message?.contains("permission") == true+ val context = LocalContext.current+ val label =+ when (state) {+ is VoiceInputState.Downloading -> "Downloading"+ VoiceInputState.Preparing,+ VoiceInputState.Idle -> "Preparing"+ VoiceInputState.Transcribing -> "Transcribing"+ VoiceInputState.Recording ->+ if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"+ else "Listening"+ is VoiceInputState.Error ->+ if (permissionNeeded) "Microphone access"+ else if (state.message == "No speech detected.") "No speech" else "Stopped"+ }+ Row(+ Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),+ verticalAlignment = Alignment.CenterVertically,+ horizontalArrangement = Arrangement.spacedBy(12.dp),+ ) {+ Text(+ label,+ style = EnsuTypography.small,+ color = EnsuColor.textMuted(),+ modifier =+ Modifier.semantics { liveRegion = LiveRegionMode.Polite }+ .then(+ if (permissionNeeded)+ Modifier.clickable(onClickLabel = "Enable microphone") {+ context.startActivity(+ Intent(+ Settings.ACTION_APPLICATION_DETAILS_SETTINGS,+ Uri.parse("package:${context.packageName}"),+ )+ )+ }+ else Modifier+ ),+ )+ if (state.isRecording)+ VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))+ else Spacer(Modifier.weight(1f))+ }+}++@Composable+internal fun VoiceInputButton(+ controller: VoiceTranscriptionController,+ enabled: Boolean,+ onOpen: () -> Unit,+) {+ val context = LocalContext.current+ var permissionRequest by rememberSaveable { mutableStateOf<Long?>(null) }+ val permission =+ rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {+ if (controller.active && permissionRequest == controller.sessionVersion)+ controller.permissionResult(it)+ permissionRequest = null+ }+ fun record() {+ if (+ ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==+ PackageManager.PERMISSION_GRANTED+ )+ controller.record()+ else {+ permissionRequest = controller.sessionVersion+ controller.permissionRequested()+ permission.launch(Manifest.permission.RECORD_AUDIO)+ }+ }+ LaunchedEffect(controller.active, controller.sessionVersion) {+ if (controller.active && controller.takeInitialRecord()) record()+ }+ val state = controller.state+ val downloading = state as? VoiceInputState.Downloading+ val preparing = state == VoiceInputState.Preparing+ val transcribing = state == VoiceInputState.Transcribing+ val capturing = state.isRecording+ val canPress = if (controller.active) capturing else enabled && controller.canOpen+ val description =+ when {+ downloading != null -> "Downloading voice model"+ preparing -> "Preparing voice input"+ transcribing -> "Transcribing"+ capturing -> "Stop dictation"+ else -> "Dictate"+ }+ IconButton(+ onClick = {+ when {+ capturing -> controller.stop()+ else -> onOpen()+ }+ },+ enabled = canPress,+ modifier = Modifier.size(44.dp).semantics { contentDescription = description },+ ) {+ when {+ downloading != null || preparing || transcribing ->+ Box(Modifier.size(28.dp), contentAlignment = Alignment.Center) {+ if (downloading?.percent != null)+ CircularProgressIndicator(+ progress = { downloading.percent.coerceIn(0, 100) / 100f },+ modifier = Modifier.size(28.dp),+ strokeWidth = 2.dp,+ color = EnsuColor.textPrimary(),+ )+ else+ CircularProgressIndicator(+ Modifier.size(28.dp),+ strokeWidth = 2.dp,+ color = EnsuColor.textPrimary(),+ )+ if (downloading != null)+ Icon(Icons.Rounded.ArrowDownward, null, Modifier.size(14.dp))+ }+ capturing ->+ Box(+ Modifier.size(32.dp).background(EnsuColor.textPrimary(), CircleShape),+ contentAlignment = Alignment.Center,+ ) {+ Icon(+ painterResource(HugeIcons.StopIcon),+ null,+ Modifier.size(14.dp),+ tint = EnsuColor.backgroundBase(),+ )+ }+ else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(20.dp))+ }+ }+}++@Composable+private fun VoiceWaveform(levels: List<Float>, modifier: Modifier) {+ val animated = levels.map {+ animateFloatAsState(it, tween(100), label = "Microphone level").value+ }+ val color = EnsuColor.textPrimary()+ Canvas(modifier.semantics { contentDescription = "Microphone activity" }) {+ val step = size.width / levels.size+ val width = minOf(3.dp.toPx(), step * 0.6f)+ animated.forEachIndexed { index, level ->+ val height = 3.dp.toPx() + (size.height - 3.dp.toPx()) * level+ drawRoundRect(+ color.copy(alpha = 0.35f + level * 0.65f),+ Offset(step * index + (step - width) / 2, (size.height - height) / 2),+ Size(width, height),+ CornerRadius(width / 2),+ )+ }+ }+}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.ktindex 6bbb678100..832c392269 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt@@ -2,6 +2,9 @@ package io.ente.ensu.deviceconst val CHAT_MIN_RAM_BYTES: Long = 1_600_000_000L+internal fun supportsLiveVoice(totalMemoryBytes: Long?): Boolean =+ totalMemoryBytes != null && totalMemoryBytes > 4L * 1024 * 1024 * 1024+internal fun chatDeviceCapability(totalMemoryBytes: Long): ChatDeviceCapability =when {totalMemoryBytes <= 0 -> ChatDeviceCapability.Unknowndiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex 3da32764c8..d97eabbea1 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -4,6 +4,9 @@ import android.os.SystemClockimport android.util.Logimport io.ente.ensu.assets.AssetStoreimport io.ente.ensu.bindings.Asset+import io.ente.ensu.bindings.DictationStopReason+import io.ente.ensu.bindings.DictationStream+import io.ente.ensu.bindings.DictationStreamCallbackimport io.ente.ensu.bindings.KnowledgeEmbeddingConfigimport io.ente.ensu.bindings.LlmChatMessage as NativeChatMessageimport io.ente.ensu.bindings.LlmChatModelMemory@@ -27,14 +30,18 @@ import io.ente.ensu.bindings.llmMemoryBudgetimport io.ente.ensu.device.AndroidDeviceCapabilityProviderimport io.ente.ensu.device.requireChatSupportedimport io.ente.ensu.settings.IS_ENSU_PACKS_ENABLED-import io.ente.ensu.toDirectByteBufferimport java.io.File+import java.util.concurrent.atomic.AtomicBooleanimport java.util.concurrent.atomic.AtomicIntegerimport kotlin.math.max+import kotlinx.coroutines.CoroutineStartimport kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.awaitCancellation+import kotlinx.coroutines.coroutineScopeimport kotlinx.coroutines.currentCoroutineContextimport kotlinx.coroutines.ensureActiveimport kotlinx.coroutines.isActive+import kotlinx.coroutines.launchimport kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContext@@ -73,7 +80,6 @@ class LlmProvider(private val activeDownloads = AtomicInteger()private val embeddingAsset = knowledgeEmbeddingModelAsset()private var chatWarmupOwner: String? = null- private var voiceOwner: String? = null@Volatile private var chatForeground = false@Volatile private var appForeground = falseprivate val memoryPressure = ModelMemoryPressure(SystemClock::elapsedRealtime)@@ -153,45 +159,45 @@ class LlmProvider(}}- suspend fun prewarmVoice(owner: String) = withModelContext {- withResources {- currentCoroutineContext().ensureActive()- if (!chatForeground || memoryPressure.suppressesOptionalWork()) return@withResources- prepareVoiceResources()- var completed = false- try {- transcriber.loadModel()- currentCoroutineContext().ensureActive()- voiceOwner = owner- completed = true- } finally {- if (!completed) unloadTranscriptionModelIfLoaded()+ suspend fun runLiveVoice(+ stream: DictationStream,+ callback: DictationStreamCallback,+ ) = coroutineScope {+ val exited = AtomicBoolean(false)+ val cancellation =+ launch(Dispatchers.Default, start = CoroutineStart.UNDISPATCHED) {+ try {+ awaitCancellation()+ } finally {+ if (!exited.get()) {+ runCatching { stream.abandon(DictationStopReason.INTERRUPTION) }+ }+ }}- }- }-- suspend fun releaseVoice(owner: String) =- withContext(ioDispatcher) {- modelLoadMutex.withLock { if (voiceOwner == owner) unloadTranscriptionModelIfLoaded() }- }-- suspend fun transcribe(inputSampleRate: UInt, pcm: ByteArray): String = withModelContext {- withResources {- currentCoroutineContext().ensureActive()- prepareVoiceResources()- try {- val result = transcriber.transcribe(inputSampleRate, pcm.toDirectByteBuffer())- currentCoroutineContext().ensureActive()- result- } finally {- unloadTranscriptionModelIfLoaded()+ try {+ withModelContext {+ withResources {+ currentCoroutineContext().ensureActive()+ prepareVoiceResources()+ try {+ stream.run(transcriber, callback)+ } finally {+ unloadTranscriptionModelIfLoaded()+ }+ }}+ } catch (error: Throwable) {+ runCatching { stream.abandon(DictationStopReason.FAILURE) }+ throw error+ } finally {+ exited.set(true)+ cancellation.cancel()}}private fun prepareVoiceResources() {releaseProjector()- if (voiceOwner == null && !canAllocate(LlmMemoryOperation.VOICE)) {+ if (!canAllocate(LlmMemoryOperation.VOICE)) {unloadModel()if (!hasRequiredWorkReserve()) throw ModelMemoryUnavailable()}@@ -238,12 +244,7 @@ class LlmProvider(val result =withResources(retainAfterCooldown = true) {currentCoroutineContext().ensureActive()- if (- !chatForeground ||- !appForeground ||- voiceOwner != null ||- !isChatModelReady(selection)- )+ if (!chatForeground || !appForeground || !isChatModelReady(selection))return@withResources ChatWarmupResult.Skippedval cooldown = memoryPressure.remainingCooldownMillis()if (cooldown > 0) return@withResources ChatWarmupResult.RetryAfter(cooldown)@@ -390,7 +391,6 @@ class LlmProvider(if (!chatForeground ||memoryPressure.suppressesOptionalWork() ||- voiceOwner != null ||loadedContextLength(selection) == null) {throw ModelMemoryDeferred()@@ -506,7 +506,7 @@ class LlmProvider(checkCancellation()deviceCapabilityProvider.chatCapability().requireChatSupported()if (!isEmbeddingModelReady()) throw EmbeddingAssetInvalid()- if (maintenance && (voiceOwner != null || memoryPressure.suppressesOptionalWork()))+ if (maintenance && memoryPressure.suppressesOptionalWork())throw ModelMemoryDeferred()unloadTranscriptionModelIfLoaded()releaseProjector()@@ -608,7 +608,6 @@ class LlmProvider(}private fun unloadTranscriptionModelIfLoaded() {- voiceOwner = nullrunCatching {transcriber.unloadModel()}diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.ktindex d1480cefd7..5ba9a19595 100644--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt@@ -7,6 +7,15 @@ import org.junit.Assert.assertTrueimport org.junit.Testclass ChatDeviceCapabilityTest {+ @Test+ fun liveVoiceRequiresMoreThanFourGiB() {+ val limit = 4L * 1024 * 1024 * 1024+ for (bytes in listOf(null, -1L, 0L, 4_000_000_000L, limit)) {+ assertFalse(supportsLiveVoice(bytes))+ }+ for (bytes in listOf(limit + 1, 8_000_000_000L)) assertTrue(supportsLiveVoice(bytes))+ }+@Testfun minimumRamBlocksOnlyKnownReadingsBelowTheFloor() {val belowMinimum = chatDeviceCapability(1_599_999_999L)diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swiftindex a916c7b661..c4ada28a33 100644--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift@@ -161,9 +161,10 @@ struct ChatView: View {viewModel.setAppForeground(newValue != .background)viewModel.notesStore.setForeground(newValue == .active)if newValue == .active {+ viewModel.voiceTranscriber.setForeground(true)viewModel.refreshModelDownloadInfo()} else {- viewModel.cancelVoiceInput()+ viewModel.voiceTranscriber.setForeground(false)}}}@@ -204,6 +205,7 @@ struct ChatView: View {}.onChange(of: viewState.showSettings) { isPresented inif isPresented {+ viewModel.voiceTranscriber.interrupt()viewState.didDismissKeyboard = trueisInputFocused = false} else if viewState.pendingSignInRequest {@@ -362,7 +364,7 @@ struct ChatView: View {}.padding(.bottom, keyboard.isVisible ? keyboard.height : 0).zIndex(1)- } else if viewModel.isModelDownloaded {+ } else {let shouldAutoFocus = shouldAutoFocusInputMessageInputView(@@ -374,7 +376,7 @@ struct ChatView: View {isDownloading: viewModel.isDownloading,editingMessage: editingMessage,isProcessingAttachments: viewModel.isProcessingAttachments,- voiceInputState: viewModel.voiceInputState,+ voiceService: viewModel.voiceTranscriber,moveCursorToEndToken: viewModel.draftCursorMoveToken,onSend: {viewState.didDismissKeyboard = true@@ -392,6 +394,11 @@ struct ChatView: View {isInputFocused = falseviewModel.toggleVoiceInput()},+ voiceInsertionCaret: viewModel.voiceInsertionCaret,+ onComposerSelection: { range, composing in+ viewModel.composerSelection = range+ viewModel.composerIsComposing = composing+ },onAddImage: { data, name inviewModel.addImageAttachment(data: data, fileName: name)},diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex a992d83679..d7711f92f7 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -71,7 +71,14 @@ final class ChatViewModel: ObservableObject {@Published private(set) var isSendPending: Bool = false@Published var isDownloading: Bool = false@Published var isProcessingAttachments: Bool = false- @Published var draftText: String = ""+ @Published var draftText: String = "" {+ didSet {+ if draftText != oldValue {+ composerRevision += 1+ voiceInsertionCaret = nil+ }+ }+ }@Published var draftAttachments: [ChatAttachment] = []private var draftImageAttachmentCount: Int {draftAttachments.filter { $0.kind == .image }.count@@ -85,7 +92,6 @@ final class ChatViewModel: ObservableObject {@Published var deviceCapability: ChatDeviceCapability = .unknown@Published var showUnsupportedDeviceDialog: Bool = false@Published var generationErrorMessage: String?- @Published var voiceInputState: VoiceInputState = .idle@Published var draftCursorMoveToken = UUID()let knowledgeStore: KnowledgeStorelet notesStore: NotesStore@@ -98,7 +104,12 @@ final class ChatViewModel: ObservableObject {private let provider: LlmProviderprivate let knowledgeEmbedding: KnowledgeEmbeddingConfigprivate let knowledgeProvider: KnowledgeProvider- private let voiceTranscriber: VoiceTranscriptionService+ let voiceTranscriber: VoiceTranscriptionService+ private var composerRevision: UInt64 = 0+ private var emptyComposerId = UUID()+ var composerSelection = NSRange(location: 0, length: 0)+ var composerIsComposing = false+ @Published var voiceInsertionCaret: Int?private var chatDb: EnsuDbprivate let attachmentsDir: URLprivate let chatDbPath: String@@ -116,7 +127,6 @@ final class ChatViewModel: ObservableObject {private var pendingSendTask: Task<Void, Never>?private var pendingSendId: UUID?private var modelDownloadTask: Task<Void, Never>?- private var voiceTransientErrorTask: Task<Void, Never>?private var sharedModelReadyTask: Task<Void, Error>?private var sharedModelReadyTaskId: UUID?private var sharedModelReadyKey: ModelReadyKey?@@ -142,9 +152,18 @@ final class ChatViewModel: ObservableObject {// Must initialize after the asset store migrates the persisted selection.self.modelSettings = ModelSettingsStore.sharedlet transcription = transcriptionModelAsset()- let transcriber = Transcriber(+ var info = utsname()+ uname(&info)+ let machine = withUnsafeBytes(of: &info.machine) { bytes in+ String(decoding: bytes.prefix { $0 != 0 }, as: UTF8.self)+ }+ let transcriber = Transcriber.withCache(modelDir: assetStore.assetDir(transcription).path,- vadModelPath: assetStore.voiceActivityModelPath().path+ vadModelPath: assetStore.voiceActivityModelPath().path,+ cacheDirectory: (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)+ .first+ ?? FileManager.default.temporaryDirectory).path,+ deviceIdentity: "\(machine)|\(ProcessInfo.processInfo.operatingSystemVersionString)")let provider = LlmProvider(assetStore: assetStore, transcriber: transcriber,@@ -220,14 +239,7 @@ final class ChatViewModel: ObservableObject {loadMessagesFromDb(for: current)}- voiceTranscriber.onTaskActivityChanged = { [weak self] active in- guard let self else { return }- if active {- self.cancelChatWarmup(releaseLoadedModel: false)- } else {- self.refreshChatWarmup()- }- }+ voiceTranscriber.onActivityChanged = { [weak self] in self?.refreshChatWarmup() }refreshDeviceCapability()refreshModelDownloadInfo()Task {@@ -260,6 +272,7 @@ final class ChatViewModel: ObservableObject {}func handleMemoryPressure() {+ voiceTranscriber.interrupt(message: "Voice input paused because memory is low.")sessionSummaryTask?.cancel()warmupSuppressed = truecancelChatWarmup()@@ -281,7 +294,7 @@ final class ChatViewModel: ObservableObject {let key = modelReadyKey(for: selection)let eligible =appForeground && chatActive && !warmupSuppressed && !isChatUnsupported- && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks+ && !voiceTranscriber.active && !voiceTranscriber.hasActiveTasksif let attempt = warmupAttempt, attempt.selection != key { cancelChatWarmup() }guard eligible, warmupAttempt == nil, !isGenerating, !isDownloading,provider.isChatModelReady(selection)@@ -307,7 +320,7 @@ final class ChatViewModel: ObservableObject {&& modelReadyKey(for: modelSettings.currentSelection()) == key&& appForeground && chatActive && !warmupSuppressed&& !isChatUnsupported && !isGenerating && !isDownloading- && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks+ && !voiceTranscriber.active && !voiceTranscriber.hasActiveTasks},operation: {try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)@@ -472,6 +485,7 @@ final class ChatViewModel: ObservableObject {return}if currentSessionId == session.id {+ cancelVoiceInput()discardUnstoredAttachments(draftAttachments)}for id in attachmentIds {@@ -501,6 +515,7 @@ final class ChatViewModel: ObservableObject {return}guard message.role == .user else { return }+ cancelVoiceInput()resetGenerationState()editingMessageId = message.iddraftText = message.text@@ -516,76 +531,45 @@ final class ChatViewModel: ObservableObject {}func toggleVoiceInput() {+ guard !isGenerating, !isDownloading, !isSendPending, editingMessageId == nil,+ !composerIsComposing, !voiceTranscriber.active, !voiceTranscriber.hasActiveTasks+ else { return }+ let range = composerSelection+ let text = draftText as NSString+ guard range.location >= 0, range.length >= 0, range.location <= text.length,+ range.length <= text.length - range.location,+ [range.location, NSMaxRange(range)].allSatisfy({+ $0 == text.length || text.rangeOfComposedCharacterSequence(at: $0).location == $0+ })+ else { return }sessionSummaryTask?.cancel()cancelChatWarmup(releaseLoadedModel: false)- if voiceInputState.isRecording {- voiceTranscriber.stopAndTranscribe(- onState: { [weak self] state in- self?.setVoiceInputState(state)- },- onTranscript: { [weak self] transcript in- self?.appendVoiceTranscript(transcript)- }- )- return- }-- guard !isGenerating,- !isDownloading,- editingMessageId == nil- else {- return+ let destination = (currentSessionId ?? emptyComposerId).uuidString+ let revision = composerRevision+ let insertion = DictationInsertion(+ destination: destination, revision: revision,+ startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))+ voiceTranscriber.onInsert = { [weak self] insertion, text in+ guard let self, self.appForeground else { return false }+ do {+ let result = try insertion.apply(+ destination: (self.currentSessionId ?? self.emptyComposerId).uuidString,+ revision: self.composerRevision, text: self.draftText, voiceText: text,+ foreground: self.appForeground, keyboardComposing: self.composerIsComposing)+ self.draftText = result.text+ self.composerRevision = result.revision+ self.voiceInsertionCaret = Int(result.caretUtf16)+ self.draftCursorMoveToken = UUID()+ return true+ } catch { return false }}-- let voiceSessionId = currentSessionId- voiceTranscriber.startRecording(- onState: { [weak self] state in- self?.setVoiceInputState(state)- },- shouldStartRecording: { [weak self] in- guard let self else { return false }- return self.currentSessionId == voiceSessionId && !self.isGenerating- && !self.isDownloading && self.editingMessageId == nil- }- )+ voiceTranscriber.open(insertion: insertion)}func cancelVoiceInput() {voiceTranscriber.cancel()- setVoiceInputState(.idle)- }-- private func setVoiceInputState(_ state: VoiceInputState) {- defer { refreshChatWarmup() }- voiceTransientErrorTask?.cancel()- voiceTransientErrorTask = nil- voiceInputState = state-- guard state.isNoSpeechError else { return }- voiceTransientErrorTask = Task { @MainActor in- do {- try await Task.sleep(nanoseconds: 10_000_000_000)- } catch {- return- }- if self.voiceInputState == state {- self.voiceInputState = .idle- }- self.voiceTransientErrorTask = nil- }- }-- private func appendVoiceTranscript(_ transcript: String) {- let cleaned = transcript.trimmingCharacters(in: .whitespacesAndNewlines)- guard !cleaned.isEmpty else { return }-- let trimmedDraft = draftText.trimmingCharacters(in: .whitespacesAndNewlines)- if trimmedDraft.isEmpty {- draftText = cleaned- } else {- draftText = "\(trimmedDraft) \(cleaned)"- }- draftCursorMoveToken = UUID()+ emptyComposerId = UUID()+ voiceInsertionCaret = nil}func addImageAttachment(data: Data, fileName: String?) {@@ -731,6 +715,7 @@ final class ChatViewModel: ObservableObject {}func sendDraft() {+ guard !voiceTranscriber.active else { return }guard !isGenerating && !isDownloading && !isSendPending else { return }guard !isChatUnsupported else {showUnsupportedDeviceDialog = true@@ -1485,7 +1470,10 @@ final class ChatViewModel: ObservableObject {let details ="session=\(userNode.sessionId.uuidString) promptLen=\(prompt.text.count) responseLen=\(snapshot.text.count) tokens=\(snapshot.tokenCount)"logger.error("Generation failed", error, details: details)- if case let ConversationError.Other(detail) = error {+ if case ConversationError.CurrentInputTooLarge = error {+ generationErrorMessage =+ "This message is too long for the model, even without earlier messages. Shorten it or split it into smaller messages. Your full text is still available to edit or copy."+ } else if case let ConversationError.Other(detail) = error {generationErrorMessage = detail} else if case ConversationError.Stale = error {generationErrorMessage = "Conversation changed. Retry the message."@@ -1607,6 +1595,9 @@ final class ChatViewModel: ObservableObject {sessionId: parent.sessionId, promptText: prompt.text,promptImageCount: prompt.imageFiles.count, currentMessageId: parent.id,limits: limits, systemPrompt: systemPrompt)+ if normalHistory.currentInputTooLarge {+ throw ConversationError.CurrentInputTooLarge+ }if normalHistory.wasTrimmed && overflowBypassMessageId != parent.id {return .overflow(inputTokens: normalHistory.inputTokens,@@ -2512,6 +2503,7 @@ final class ChatViewModel: ObservableObject {let inputTokens: Intlet inputBudget: Intlet wasTrimmed: Bool+ let currentInputTooLarge: Bool}private struct GenerationLimits {@@ -2580,7 +2572,8 @@ final class ChatViewModel: ObservableObject {if remaining <= 0 || historyMessages.isEmpty {return HistorySelection(messages: [], inputTokens: inputTokens, inputBudget: inputBudget,- wasTrimmed: inputTokens > inputBudget)+ wasTrimmed: inputTokens > inputBudget,+ currentInputTooLarge: systemTokens + promptTokens > inputBudget)}let quantum = max(1, inputBudget / 4)@@ -2610,7 +2603,7 @@ final class ChatViewModel: ObservableObject {return HistorySelection(messages: selected, inputTokens: inputTokens, inputBudget: inputBudget,- wasTrimmed: inputTokens > inputBudget)+ wasTrimmed: inputTokens > inputBudget, currentInputTooLarge: false)}private func resolveMaxOutputTokens(contextLength: Int) -> Int {diff --git a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift b/apple/apps/ensu/Ensu/Chat/MessageInputView.swiftindex 17f78d90bd..f9bf3bd0d8 100644--- a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift+++ b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift@@ -12,12 +12,14 @@ struct MessageInputView: View {let isDownloading: Boollet editingMessage: RenderedChatMessage?let isProcessingAttachments: Bool- let voiceInputState: VoiceInputState+ @ObservedObject var voiceService: VoiceTranscriptionServicelet moveCursorToEndToken: UUIDlet onSend: () -> Voidlet onStop: () -> Voidlet onCancelEdit: () -> Voidlet onVoiceInput: () -> Void+ let voiceInsertionCaret: Int?+ let onComposerSelection: (NSRange, Bool) -> Voidlet onAddImage: (Data, String?) -> Voidlet onAddDocument: (URL) -> Voidlet onRemoveAttachment: (ChatAttachment) -> Void@@ -38,7 +40,8 @@ struct MessageInputView: View {private var isSendEnabled: Bool {let hasContent =!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !attachments.isEmpty- return hasContent && !isDownloading && !isSendPending && !voiceInputState.blocksSend+ return hasContent && !isDownloading && !isSendPending+ && !voiceService.active}private var isImageAttachmentLimitReached: Bool {@@ -46,6 +49,9 @@ struct MessageInputView: View {}var body: some View {+ let voicePreview = voiceService.composerPreview(original: text)+ let composerText = voicePreview?.text ?? text+ZStack(alignment: .bottomTrailing) {VStack(alignment: .leading, spacing: EnsuSpacing.sm) {if let editingMessage {@@ -92,37 +98,11 @@ struct MessageInputView: View {.padding(.horizontal, EnsuSpacing.sm)}- if let voiceStatus = voiceInputState.statusText {- HStack(spacing: EnsuSpacing.sm) {- switch voiceInputState {- case .downloading, .transcribing:- ProgressView()- .scaleEffect(0.8)- case .recording:- Image("Mic02Icon")- .resizable()- .scaledToFit()- .frame(width: 14, height: 14)- .foregroundStyle(EnsuColor.stopButton)- case .error:- Image(systemName: "exclamationmark.circle")- .font(.system(size: 12, weight: .semibold))- .foregroundStyle(EnsuColor.stopButton)- case .idle:- EmptyView()- }- Text(voiceStatus)- .font(EnsuTypography.small)- .foregroundStyle(- voiceInputState.isError- ? EnsuColor.stopButton : EnsuColor.textMuted)- Spacer()- }- .padding(.horizontal, EnsuSpacing.sm)- }+ VoiceInputControls(service: voiceService)+ .padding(.horizontal, EnsuSpacing.pageHorizontal)HStack(alignment: .bottom, spacing: EnsuSpacing.xs) {- if supportsImageAttachments {+ if supportsImageAttachments && !voiceService.active {ActionButton(icon: "ImagePlusIcon", tooltip: "Add photos",color: EnsuColor.textPrimary, size: 32@@ -154,7 +134,10 @@ struct MessageInputView: View {}MessageTextView(- text: $text,+ text: Binding(+ get: { composerText },+ set: { if !voiceService.active { text = $0 } }+ ),isFocused: $isFocused,moveCursorToEndToken: moveCursorToEndToken,onSubmit: {@@ -164,11 +147,16 @@ struct MessageInputView: View {hideKeyboard()onDismissKeyboard()}- }+ },+ insertionCaret: voicePreview.map { Int($0.caretUtf16) }+ ?? voiceInsertionCaret,+ onSelectionChange: onComposerSelection,+ readOnly: voiceService.active,+ maximumLines: voiceService.active ? 8 : 5).padding(.vertical, 6).overlay(alignment: .topLeading) {- if text.isEmpty {+ if composerText.isEmpty {Text(placeholder).font(EnsuTypography.message).foregroundStyle(Color(uiColor: .placeholderText))@@ -184,49 +172,16 @@ struct MessageInputView: View {}if editingMessage == nil {- let isVoiceBusy = voiceInputState.isTranscriptionBusy- let canUseVoice =- voiceInputState.isRecording- || (!isGenerating && !isDownloading && !isVoiceBusy)-- Button {- if voiceInputState.isRecording {- hapticWarning()- } else {- hapticTap()- isFocused = false- hideKeyboard()- onDismissKeyboard()- }+ VoiceInputButton(+ service: voiceService,+ enabled: !isGenerating && !isDownloading && !isSendPending+ ) {+ hapticTap()+ isFocused = false+ hideKeyboard()+ onDismissKeyboard()onVoiceInput()- } label: {- if isVoiceBusy {- ProgressView()- .scaleEffect(0.75)- .frame(width: 32, height: 32)- } else if voiceInputState.isRecording {- Image("StopIcon")- .resizable()- .scaledToFit()- .frame(- width: composerIconSize, height: composerIconSize- )- .foregroundStyle(EnsuColor.sendButton)- .frame(width: 32, height: 32)- } else {- Image("Mic02Icon")- .resizable()- .scaledToFit()- .frame(width: composerIconSize, height: composerIconSize)- .foregroundStyle(- canUseVoice- ? EnsuColor.textPrimary : EnsuColor.textMuted- )- .frame(width: 32, height: 32)- }}- .disabled(!canUseVoice)- .padding(.leading, EnsuSpacing.xs)}Button {diff --git a/apple/apps/ensu/Ensu/Chat/MessageTextView.swift b/apple/apps/ensu/Ensu/Chat/MessageTextView.swiftindex f1e9790a83..39529ca092 100644--- a/apple/apps/ensu/Ensu/Chat/MessageTextView.swift+++ b/apple/apps/ensu/Ensu/Chat/MessageTextView.swift@@ -6,6 +6,10 @@ struct MessageTextView: UIViewRepresentable {@Binding var isFocused: Boollet moveCursorToEndToken: UUIDlet onSubmit: () -> Void+ var insertionCaret: Int? = nil+ var onSelectionChange: ((NSRange, Bool) -> Void)? = nil+ var readOnly = false+ var maximumLines: CGFloat = 5func makeCoordinator() -> Coordinator {Coordinator(self)@@ -32,13 +36,23 @@ struct MessageTextView: UIViewRepresentable {func updateUIView(_ uiView: UITextView, context: Context) {context.coordinator.parent = self+ uiView.isEditable = !readOnlyif uiView.text != text {uiView.undoManager?.removeAllActions()uiView.text = text+ if readOnly, !uiView.isDragging, !uiView.isDecelerating {+ let caret = min(+ insertionCaret ?? (text as NSString).length, (text as NSString).length)+ uiView.selectedRange = NSRange(location: caret, length: 0)+ uiView.scrollRangeToVisible(uiView.selectedRange)+ }}if context.coordinator.lastCursorToken != moveCursorToEndToken {context.coordinator.lastCursorToken = moveCursorToEndToken- uiView.selectedRange = NSRange(location: (text as NSString).length, length: 0)+ uiView.selectedRange = NSRange(+ location: min(+ insertionCaret ?? (text as NSString).length, (text as NSString).length),+ length: 0)}if isFocused != uiView.isFirstResponder {DispatchQueue.main.async { [weak uiView, weak coordinator = context.coordinator] in@@ -55,12 +69,14 @@ struct MessageTextView: UIViewRepresentable {func sizeThatFits(_ proposal: ProposedViewSize, uiView: UITextView, context: Context) -> CGSize?{guard let width = proposal.width, width > 0, width.isFinite else { return nil }- return CGSize(width: width, height: Self.height(for: text, width: width))+ return CGSize(+ width: width,+ height: Self.height(for: text, width: width, maximumLines: maximumLines))}- private static func height(for text: String, width: CGFloat) -> CGFloat {+ private static func height(for text: String, width: CGFloat, maximumLines: CGFloat) -> CGFloat {let font = UIFont.systemFont(ofSize: 15)- let maximumHeight = ceil(font.lineHeight * 5)+ let maximumHeight = ceil(font.lineHeight * maximumLines)let scalars = text.unicodeScalars.prefix(1025)guard scalars.count <= 1024 else { return maximumHeight }let sample = String(String.UnicodeScalarView(scalars)) + "\u{200B}"@@ -97,6 +113,11 @@ struct MessageTextView: UIViewRepresentable {func textViewDidChange(_ textView: UITextView) {parent.text = textView.text+ parent.onSelectionChange?(textView.selectedRange, textView.markedTextRange != nil)+ }++ func textViewDidChangeSelection(_ textView: UITextView) {+ parent.onSelectionChange?(textView.selectedRange, textView.markedTextRange != nil)}func textViewDidBeginEditing(_ textView: UITextView) {diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swiftnew file mode 100644index 0000000000..5ba2e8d49a--- /dev/null+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift@@ -0,0 +1,121 @@+import SwiftUI++struct VoiceInputControls: View {+ @ObservedObject var service: VoiceTranscriptionService+ @Environment(\.openURL) private var openURL+ @Environment(\.accessibilityReduceMotion) private var reduceMotion++ private var status: String {+ switch service.state {+ case .downloading: return "Downloading"+ case .preparing: return "Preparing"+ case .transcribing: return "Transcribing"+ case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"+ case .error(let message):+ if message.contains("permission") { return "Microphone access" }+ return message == "No speech detected." ? "No speech" : "Stopped"+ case .idle: return "Preparing"+ }+ }++ var body: some View {+ if service.active || service.state.isError {+ HStack(spacing: 12) {+ Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)+ .accessibilityHint(service.notice ?? "")+ .onTapGesture {+ if case .error(let message) = service.state,+ message.contains("permission"),+ let url = URL(string: UIApplication.openSettingsURLString)+ {+ openURL(url)+ }+ }+ if service.state.isRecording {+ waveform.frame(height: 24)+ } else {+ Spacer(minLength: 0)+ }+ }+ }+ }++ private var waveform: some View {+ GeometryReader { geometry in+ HStack(alignment: .center, spacing: 0) {+ ForEach(service.audioLevels.indices, id: \.self) { index in+ let level = service.audioLevels[index]+ Capsule()+ .fill(EnsuColor.textPrimary.opacity(0.35 + Double(level) * 0.65))+ .frame(+ width: min(3, geometry.size.width / 48), height: 3 + CGFloat(level) * 21+ )+ .frame(+ width: geometry.size.width / CGFloat(service.audioLevels.count),+ height: 24)+ }+ }+ .animation(reduceMotion ? nil : .easeOut(duration: 0.1), value: service.audioLevels)+ }+ .accessibilityElement(children: .ignore)+ .accessibilityLabel("Microphone activity")+ }+}++struct VoiceInputButton: View {+ @ObservedObject var service: VoiceTranscriptionService+ let enabled: Bool+ let onOpen: () -> Void++ private var canPress: Bool {+ service.active ? service.state.isRecording : enabled && !service.hasActiveTasks+ }++ private var label: String {+ switch service.state {+ case .downloading: return "Downloading voice model"+ case .preparing: return "Preparing voice input"+ case .transcribing: return "Transcribing"+ case .recording: return "Stop dictation"+ case .idle, .error: return "Dictate"+ }+ }++ var body: some View {+ Button {+ if service.state.isRecording { service.stop() } else { onOpen() }+ } label: {+ ZStack {+ switch service.state {+ case .downloading(let percent):+ if let percent {+ Circle().stroke(EnsuColor.textMuted.opacity(0.2), lineWidth: 2)+ Circle().trim(from: 0, to: CGFloat(min(100, max(0, percent))) / 100)+ .stroke(+ EnsuColor.textPrimary,+ style: StrokeStyle(lineWidth: 2, lineCap: .round)+ )+ .rotationEffect(.degrees(-90))+ } else {+ ProgressView().controlSize(.small)+ }+ Image(systemName: "arrow.down").font(.system(size: 11, weight: .semibold))+ case .preparing, .transcribing:+ ProgressView().controlSize(.small)+ case .recording:+ Circle().fill(EnsuColor.textPrimary).frame(width: 32, height: 32)+ RoundedRectangle(cornerRadius: 2).fill(EnsuColor.backgroundBase)+ .frame(width: 12, height: 12)+ case .idle, .error:+ Image("Mic02Icon").resizable().scaledToFit().frame(width: 20, height: 20)+ }+ }+ .frame(width: 28, height: 28)+ .frame(width: 44, height: 44)+ .contentShape(Rectangle())+ }+ .foregroundStyle(canPress ? EnsuColor.textPrimary : EnsuColor.textMuted)+ .disabled(!canPress)+ .accessibilityLabel(label)+ }+}diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 3748339cd9..029d4c1a4d 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -1,45 +1,17 @@import Foundationimport AVFoundation+import Combineenum VoiceInputState: Equatable {case idle+ case preparingcase recordingcase downloading(percent: Int?)case transcribingcase error(String)var isRecording: Bool {- if case .recording = self {- return true- }- return false- }-- var isWorking: Bool {- switch self {- case .recording, .downloading, .transcribing:- return true- case .idle, .error:- return false- }- }-- var isTranscriptionBusy: Bool {- switch self {- case .downloading, .transcribing:- return true- case .idle, .recording, .error:- return false- }- }-- var blocksSend: Bool {- switch self {- case .recording, .transcribing:- return true- case .idle, .downloading, .error:- return false- }+ self == .recording}var isError: Bool {@@ -49,394 +21,493 @@ enum VoiceInputState: Equatable {return false}- var isNoSpeechError: Bool {- switch self {- case let .error(message):- return message == "No speech detected." || message == "No speech captured."- case .idle, .recording, .downloading, .transcribing:- return false- }- }-- var statusText: String? {- switch self {- case .idle:- return nil- case .recording:- return "Listening..."- case let .downloading(percent):- if let percent {- return "Downloading voice model... (\(percent)%)"- }- return "Downloading voice model..."- case .transcribing:- return nil- case let .error(message):- return message- }- }}@MainActor-final class VoiceTranscriptionService {- typealias StateHandler = @MainActor @Sendable (VoiceInputState) -> Void- typealias TranscriptHandler = @MainActor @Sendable (String) -> Void-- var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?+final class VoiceTranscriptionService: ObservableObject {+ @Published private(set) var active = false {+ didSet { onActivityChanged?() }+ }+ @Published private(set) var snapshot: DictationSnapshot?+ @Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)+ @Published private(set) var notice: String?+ @Published private(set) var state: VoiceInputState = .idle+ var onActivityChanged: (@MainActor () -> Void)?+ var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?private let provider: LlmProviderprivate let assetStore: AssetStoreprivate let modelAssets: [Asset]- private var transcriptionTask: Task<Void, Never>?- private var preloadTask: Task<Void, Never>?- private var voiceOwner = UUID()- private var activeVoiceTaskId = UUID()- private var activeDownloadId: UUID?- private var activeTaskCount = 0-- var hasActiveTasks: Bool { activeTaskCount > 0 }-- private let recorder = PcmAudioRecorder()+ private let usesLiveDictation: Bool+ private var session: DictationSession?+ private var stream: DictationStream?+ private var recorder: LivePcmRecorder?+ private var task: Task<Void, Never>?+ private var finishingWatchdog: Task<Void, Never>?+ private var attemptFinished = false+ private var insertion: DictationInsertion?+ private var generation = UUID()+ private var foreground = true+ private var awaitingPermission = false+ private var startOnForeground = false+ private let notifications = VoiceNotificationTokens()+ @Published private(set) var hasActiveTasks = false {+ didSet { onActivityChanged?() }+ }+ private func diagnostic(_ event: String) {+ #if DEBUG+ EnsuLogging.shared.logger("EnsuVoice").info(event)+ #endif+ }init(provider: LlmProvider, assetStore: AssetStore) {self.provider = providerself.assetStore = assetStore- self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]+ usesLiveDictation = supportsLiveVoice()+ modelAssets = [+ transcriptionModelAsset(),+ voiceActivityModelAsset(),+ ]+ for name in [+ AVAudioSession.interruptionNotification, AVAudioSession.routeChangeNotification,+ AVAudioSession.mediaServicesWereResetNotification,+ ] {+ notifications.values.append(+ NotificationCenter.default.addObserver(forName: name, object: nil, queue: .main) {+ [weak self] notification in+ let route = notification.name == AVAudioSession.routeChangeNotification+ let reason = notification.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt+ Task { @MainActor [weak self] in+ guard let self,+ self.active,+ !route || self.recorder?.isRecording == true || self.state.isRecording+ else { return }+ let audio = AVAudioSession.sharedInstance()+ if route,+ reason == AVAudioSession.RouteChangeReason.categoryChange.rawValue,+ audio.category == .playAndRecord, audio.mode == .measurement+ {+ return+ }+ self.diagnostic(+ "audio_interruption route=\(route) reason=\(reason ?? 0)")+ self.interrupt()+ }+ })+ }}- private func launchVoiceTask(- priority: TaskPriority, operation: @escaping @Sendable () async -> Void- ) -> Task<Void, Never> {- activeTaskCount += 1- if activeTaskCount == 1 { onTaskActivityChanged?(true) }- let task = Task.detached(priority: priority, operation: operation)- Task { [weak self] in- await task.value- guard let self else { return }- self.activeTaskCount -= 1- if self.activeTaskCount == 0 { self.onTaskActivityChanged?(false) }+ func open(insertion: DictationInsertion) {+ guard !active, !hasActiveTasks else { return }+ generation = UUID()+ self.insertion = insertion+ attemptFinished = false+ session = DictationSession(live: usesLiveDictation)+ snapshot = session?.snapshot()+ active = true+ state = .idle+ notice = nil+ record()+ }++ func setForeground(_ active: Bool) {+ foreground = active+ if active {+ if startOnForeground {+ startOnForeground = false+ record()+ } else if let owner = session {+ refresh(owner, token: generation)+ }+ } else if !awaitingPermission, !startOnForeground {+ interrupt()}- return task}- func startRecording(- onState: @escaping StateHandler,- shouldStartRecording: @escaping @MainActor @Sendable () -> Bool = { true }- ) {- guard !recorder.isRecording else { return }+ func composerPreview(original: String) -> DictationInsertionResult? {+ guard active, let insertion, let snapshot else { return nil }+ return try? insertion.preview(+ text: original, voiceText: snapshot.text + snapshot.hypothesis)+ }- let session = AVAudioSession.sharedInstance()- switch session.recordPermission {- case .granted:- prepareModelAndStartRecording(- onState: onState, shouldStartRecording: shouldStartRecording)- case .denied:- onState(.error("Microphone permission is required for voice input."))+ func record() {+ guard active, foreground, !hasActiveTasks, !attemptFinished, let owner = session+ else { return }+ let token = generation+ switch AVAudioSession.sharedInstance().recordPermission {case .undetermined:- session.requestRecordPermission { @Sendable [weak self] granted in- Task { @MainActor in- guard let self else { return }+ guard !awaitingPermission else { return }+ awaitingPermission = true+ AVAudioSession.sharedInstance().requestRecordPermission {+ @Sendable [weak self] granted in+ Task { @MainActor [weak self] in+ guard let self, self.generation == token else { return }+ guard self.active else { return }+ self.awaitingPermission = falseif granted {- self.prepareModelAndStartRecording(- onState: onState,- shouldStartRecording: shouldStartRecording- )+ if self.foreground { self.record() } else { self.startOnForeground = true }} else {- onState(.error("Microphone permission is required for voice input."))+ self.cancel()+ self.state = .error(+ "Microphone permission is required. Enable it in Settings.")}}}- @unknown default:- onState(.error("Microphone permission is required for voice input."))- }- }-- func stopAndTranscribe(- onState: @escaping StateHandler,- onTranscript: @escaping TranscriptHandler- ) {- guard recorder.isRecording else { return }- onState(.transcribing)- let recording = recorder.stop()-- guard recording.pcm.count >= minimumRecordingBytes(sampleRate: recording.sampleRate) else {- releaseVoice()- onState(.error("No speech captured."))return+ case .denied:+ cancel()+ state = .error("Microphone permission is required. Enable it in Settings.")+ return+ case .granted: break+ @unknown default: return}-- let taskId = beginVoiceTask()- let downloadId = beginDownload(taskId: taskId)- let provider = provider- let sampleRate = recording.sampleRate- let pcm = recording.pcm- let owner = voiceOwner-- transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in- defer { Task.detached { await provider.releaseVoice(owner: owner) } }- do {- try await self?.downloadModelsIfNeeded(- taskId: taskId,- downloadId: downloadId,- onState: onState- )-- if Task.isCancelled { return }- let isActive = await MainActor.run { [weak self] in- self?.finishDownload(downloadId: downloadId)- return self?.isVoiceTaskActive(taskId) == true+ hasActiveTasks = true+ notice = nil+ state = .preparing+ task = Task { [weak self] in+ guard let self else { return }+ defer {+ self.hasActiveTasks = false+ if self.generation == token {+ self.finishingWatchdog?.cancel()+ self.finishingWatchdog = nil+ self.recorder?.stop()+ self.recorder = nil+ self.stream = nil+ self.attemptFinished = true+ self.refresh(owner, token: token)}- guard isActive else { return }- await MainActor.run { [weak self] in- guard self?.isVoiceTaskActive(taskId) == true else { return }- onState(.transcribing)+ }+ do {+ if !self.modelAssets.allSatisfy({ self.assetStore.isDownloaded($0) }) {+ self.state = .downloading(percent: nil)+ try await self.assetStore.download(assets: self.modelAssets) {+ [weak self] progress in+ Task { @MainActor [weak self] in+ guard let self, self.generation == token,+ case .downloading = self.state+ else { return }+ self.state = .downloading(percent: Int(progress.percentage))+ }+ }}- let preload = await MainActor.run { self?.preloadTask }- await preload?.valuetry Task.checkCancellation()- let transcript = try await provider.transcribe(- inputSampleRate: sampleRate, pcmLe: pcm- )- .trimmingCharacters(in: .whitespacesAndNewlines)-- if Task.isCancelled { return }- await MainActor.run { [weak self] in- guard self?.isVoiceTaskActive(taskId) == true else { return }- if transcript.isEmpty {- onState(.error("No speech detected."))- } else {- onTranscript(transcript)- onState(.idle)+ guard self.generation == token, self.foreground else { return }+ self.state = .preparing+ let recorder = try LivePcmRecorder()+ self.recorder = recorder+ let sampleRate = recorder.sampleRate+ let stream = try DictationStream(+ session: owner, sampleRate: sampleRate, channels: 1)+ self.stream = stream+ let delivery = LiveCaptureDelivery(+ stream: stream, sampleRate: sampleRate,+ level: { [weak self] level in+ Task { @MainActor [weak self] in+ guard let self, self.generation == token, self.state.isRecording else {+ return+ }+ self.audioLevels = Array(self.audioLevels.dropFirst()) + [level]+ }}- }- } catch is CancellationError {- return- } catch {- if case AssetDownloadError.Cancelled = error { return }- if case LlmError.Cancelled = error { return }- await MainActor.run { [weak self] in- self?.finishDownload(downloadId: downloadId)- guard self?.isVoiceTaskActive(taskId) == true else { return }- onState(- .error(- error is ModelMemoryUnavailable- ? error.localizedDescription : "Could not transcribe voice input."))- }- }- }- }-- func cancel() {- releaseVoice()- transcriptionTask?.cancel()- transcriptionTask = nil- activeVoiceTaskId = UUID()- activeDownloadId = nil- if recorder.isRecording {- _ = recorder.stop()- }- }-- private func prepareModelAndStartRecording(- onState: @escaping StateHandler,- shouldStartRecording: @escaping @MainActor @Sendable () -> Bool- ) {- let taskId = beginVoiceTask()- let downloadId = beginDownload(taskId: taskId)-- transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in- do {- try await self?.downloadModelsIfNeeded(- taskId: taskId,- downloadId: downloadId,- onState: onState- )-- if Task.isCancelled { return }- await MainActor.run { [weak self] in- self?.finishDownload(downloadId: downloadId)- guard let self, self.isVoiceTaskActive(taskId) else { return }- guard shouldStartRecording() else {- onState(.idle)- return+ ) {+ [weak self] in+ Task { @MainActor [weak self] in+ guard let self, self.generation == token else { return }+ self.interrupt(+ message:+ "Audio capture interrupted. Some words may be missing; review the text."+ )}- self.beginRecording(onState: onState)- if self.recorder.isRecording { self.preloadTranscriptionModel() }}+ let callback = LiveVoiceCallback(+ ready: { [weak self] in+ Task { @MainActor [weak self] in+ let value = owner.snapshot()+ guard let self, self.generation == token, self.foreground,+ self.active, self.stream === stream, value.workerActive,+ value.phase == .listening+ else { return }+ do {+ try recorder.start(delivery: delivery)+ self.state = .recording+ self.audioLevels = Array(repeating: 0, count: 24)+ self.notice = nil+ self.refresh(owner, token: token)+ } catch {+ self.interrupt(+ message:+ "Could not start the microphone. Tap the microphone to start again."+ )+ }+ }+ },+ change: { [weak self] in+ Task { @MainActor [weak self] in self?.refresh(owner, token: token) }+ },+ closed: { [weak self] in+ delivery.close()+ Task { @MainActor [weak self] in+ guard let self, self.generation == token, self.stream === stream+ else { return }+ recorder.stop()+ self.watchFinalization(owner, stream: stream, token: token)+ }+ }+ )+ try await self.provider.runLiveVoice(stream: stream, callback: callback)} catch is CancellationError {- return} catch {- if case AssetDownloadError.Cancelled = error { return }- if case LlmError.Cancelled = error { return }- await MainActor.run { [weak self] in- self?.finishDownload(downloadId: downloadId)- guard self?.isVoiceTaskActive(taskId) == true else { return }- onState(.error("Voice model download failed."))+ if let error = error as? DictationError, case .InvalidOperation(let detail) = error+ {+ self.diagnostic("failure \(detail)")+ } else {+ self.diagnostic("failure \(type(of: error))")}- }- }- }-- private nonisolated func downloadModelsIfNeeded(- taskId: UUID,- downloadId: UUID,- onState: @escaping StateHandler- ) async throws {- let assets = modelAssets- let assetStore = self.assetStore- if assets.allSatisfy({ assetStore.isDownloaded($0) }) {- return- }- await MainActor.run { [weak self] in- guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else {- return- }- onState(.downloading(percent: nil))- }- try await assetStore.download(assets: assets) { [weak self] progress in- let percent = min(max(Int(progress.percentage), 0), 100)- Task { @MainActor [weak self] in- guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else {- return+ if self.generation == token, self.notice == nil {+ self.notice = "Voice input stopped. Review the available text or try again."}- onState(.downloading(percent: percent))}}}- private func beginVoiceTask() -> UUID {- transcriptionTask?.cancel()- let taskId = UUID()- activeVoiceTaskId = taskId- activeDownloadId = nil- return taskId- }-- private func preloadTranscriptionModel() {- let provider = provider- voiceOwner = UUID()- let owner = voiceOwner- preloadTask = launchVoiceTask(priority: .utility) {- try? await provider.prewarmVoice(owner: owner)+ private func refresh(_ owner: DictationSession, token: UUID) {+ guard generation == token, active else { return }+ let value = owner.snapshot()+ snapshot = value+ guard attemptFinished else {+ if value.phase == .finishing { state = .transcribing }+ return}- }-- private func releaseVoice() {- preloadTask?.cancel()- preloadTask = nil- let owner = voiceOwner- voiceOwner = UUID()- let provider = provider- _ = launchVoiceTask(priority: .utility) { await provider.releaseVoice(owner: owner) }- }-- private func isVoiceTaskActive(_ taskId: UUID) -> Bool {- activeVoiceTaskId == taskId- }-- private func beginDownload(taskId: UUID) -> UUID {- let downloadId = UUID()- if isVoiceTaskActive(taskId) {- activeDownloadId = downloadId+ guard foreground else { return }+ let text = value.text + value.hypothesis+ let hasText = !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty+ let added = hasText && insertion.map { onInsert?($0, text) == true } == true+ let completedState: VoiceInputState+ if added {+ completedState = value.incomplete ? .error("Voice input stopped.") : .idle+ } else {+ completedState = .error(+ hasText ? "Voice input stopped." : notice ?? "No speech detected.")}- return downloadId+ active = false+ session = nil+ snapshot = nil+ insertion = nil+ onInsert = nil+ audioLevels = Array(repeating: 0, count: 24)+ state = completedState}- private func finishDownload(downloadId: UUID) {- if activeDownloadId == downloadId {- activeDownloadId = nil+ private func watchFinalization(_ owner: DictationSession, stream: DictationStream, token: UUID)+ {+ guard token == generation, active, self.stream === stream, owner.snapshot().workerActive,+ finishingWatchdog == nil+ else { return }+ finishingWatchdog = Task { @MainActor [weak self] in+ do { try await Task.sleep(for: .seconds(10)) } catch { return }+ guard let self, self.generation == token, self.active, self.stream === stream,+ owner.snapshot().workerActive+ else { return }+ self.recorder?.stop()+ try? stream.abandon(reason: .failure)+ self.task?.cancel()+ self.snapshot = owner.snapshot()+ self.notice =+ "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."}}- private func isDownloadActive(taskId: UUID, downloadId: UUID) -> Bool {- isVoiceTaskActive(taskId) && activeDownloadId == downloadId+ func stop() {+ guard let owner = session else { return }+ let reason: DictationStopReason = .done+ _ = try? owner.stop(reason: reason)+ snapshot = owner.snapshot()+ state = .transcribing+ let stream = stream+ if let stream { watchFinalization(owner, stream: stream, token: generation) }+ recorder?.stop(afterDelivery: { try? stream?.stop(reason: reason) })+ if recorder == nil { try? stream?.stop(reason: reason) }}- private func beginRecording(onState: @escaping StateHandler) {- do {- try recorder.start()- onState(.recording)- } catch {- onState(.error("Could not record microphone audio."))+ func interrupt(message: String = "Voice input stopped.") {+ guard active, !attemptFinished else { return }+ if !hasActiveTasks, stream == nil {+ cancel()+ return}+ recorder?.stop()+ try? stream?.abandon(reason: .interruption)+ task?.cancel()+ notice = message+ state = .transcribing}- private func minimumRecordingBytes(sampleRate: UInt32) -> Int {- Int(sampleRate) / 4 * 2+ func cancel() {+ awaitingPermission = false+ startOnForeground = false+ generation = UUID()+ finishingWatchdog?.cancel()+ finishingWatchdog = nil+ recorder?.stop()+ try? stream?.abandon(reason: .discard)+ task?.cancel()+ active = false+ state = .idle+ snapshot = nil+ session = nil+ insertion = nil+ stream = nil+ recorder = nil+ onInsert = nil}}-private struct VoiceRecording {- let sampleRate: UInt32- let pcm: Data+private final class LiveVoiceCallback: DictationStreamCallback, Sendable {+ let ready: @Sendable () -> Void+ let change: @Sendable () -> Void+ let closed: @Sendable () -> Void+ init(+ ready: @escaping @Sendable () -> Void,+ change: @escaping @Sendable () -> Void,+ closed: @escaping @Sendable () -> Void+ ) {+ self.ready = ready; self.change = change; self.closed = closed+ }+ func onReady() { ready() }+ func onChange() { change() }+ func onCaptureClosed() { closed() }}-private final class PcmAudioRecorder {+@MainActor+private final class LivePcmRecorder {private let engine = AVAudioEngine()- private let lock = NSLock()- private var pcm = Data()- private var sampleRate: UInt32 = 16_000-- var isRecording: Bool {- engine.isRunning- }-- func start() throws {- lock.lock()- pcm.removeAll(keepingCapacity: true)- lock.unlock()+ private var delivery: LiveCaptureDelivery?+ private var installed = false+ let sampleRate: UInt32+ var isRecording: Bool { engine.isRunning }- let session = AVAudioSession.sharedInstance()- try session.setCategory(+ init() throws {+ let audio = AVAudioSession.sharedInstance()+ try audio.setCategory(.playAndRecord, mode: .measurement, options: [.allowBluetoothHFP, .defaultToSpeaker])- try session.setActive(true)+ try audio.setActive(true)+ sampleRate = UInt32(engine.inputNode.outputFormat(forBus: 0).sampleRate.rounded())+ guard (8000...48000).contains(sampleRate) else {+ try? audio.setActive(false, options: .notifyOthersOnDeactivation)+ throw CocoaError(.featureUnsupported)+ }+ }+ func start(delivery: LiveCaptureDelivery) throws {let input = engine.inputNodelet format = input.outputFormat(forBus: 0)- sampleRate = UInt32(format.sampleRate.rounded())-- input.removeTap(onBus: 0)- input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak self] buffer, _ in- self?.append(buffer)+ guard UInt32(format.sampleRate.rounded()) == sampleRate else {+ throw CocoaError(.featureUnsupported)}-+ self.delivery = delivery+ input.installTap(onBus: 0, bufferSize: AVAudioFrameCount(sampleRate / 10), format: format) {+ @Sendable buffer, _ in+ delivery.accept(buffer)+ }+ installed = trueengine.prepare()try engine.start()}- func stop() -> VoiceRecording {- engine.inputNode.removeTap(onBus: 0)+ func stop(afterDelivery: (@Sendable () -> Void)? = nil) {+ if installed { engine.inputNode.removeTap(onBus: 0); installed = false }+ delivery?.close()engine.stop()try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)-- lock.lock()- let result = VoiceRecording(sampleRate: sampleRate, pcm: pcm)- pcm.removeAll(keepingCapacity: true)- lock.unlock()- return result+ if let afterDelivery {+ if let delivery { delivery.drain(afterDelivery) } else { afterDelivery() }+ }}+}- private func append(_ buffer: AVAudioPCMBuffer) {- guard let channels = buffer.floatChannelData else { return }- let frameCount = Int(buffer.frameLength)- let channelCount = max(Int(buffer.format.channelCount), 1)- guard frameCount > 0 else { return }+private final class VoiceNotificationTokens: @unchecked Sendable {+ var values: [NSObjectProtocol] = []+ deinit { for value in values { NotificationCenter.default.removeObserver(value) } }+}- var chunk = Data(capacity: frameCount * 2)- for frame in 0..<frameCount {- var sample: Float = 0- for channel in 0..<channelCount {- sample += channels[channel][frame]- }- sample /= Float(channelCount)- let clamped = min(max(sample, -1), 1)- var intSample = Int16(clamped * Float(Int16.max)).littleEndian- withUnsafeBytes(of: &intSample) { bytes in- chunk.append(contentsOf: bytes)- }- }+private final class LiveCaptureDelivery: @unchecked Sendable {+ private let queue = DispatchQueue(label: "io.ente.ensu.voice.capture", qos: .userInitiated)+ private let lock = NSLock()+ private var accepting = true+ private var framesPending = 0+ private var chunksPending = 0+ private let stream: DictationStream+ private let rate: Int+ private let interrupted: @Sendable () -> Void+ private let level: @Sendable (Float) -> Void+ private var meterFrames = 0+ private var meterEnergy = 0.0++ init(+ stream: DictationStream, sampleRate: UInt32,+ level: @escaping @Sendable (Float) -> Void, interrupted: @escaping @Sendable () -> Void+ ) {+ self.stream = stream; rate = Int(sampleRate); self.interrupted = interrupted+ self.level = level+ }+ func close() { lock.lock(); accepting = false; lock.unlock() }+ func drain(_ completion: @escaping @Sendable () -> Void) { queue.async(execute: completion) }+ func accept(_ buffer: AVAudioPCMBuffer) {+ let frames = Int(buffer.frameLength)+ guard frames > 0 else { return }lock.lock()- pcm.append(chunk)+ guard accepting else { lock.unlock(); return }+ guard frames <= rate, framesPending + frames <= rate * 2, chunksPending < 128,+ let channels = buffer.floatChannelData, buffer.format.channelCount > 0+ else {+ accepting = false+ lock.unlock()+ queue.async { [self] in+ try? stream.abandon(reason: .failure); interrupted()+ }+ return+ }+ framesPending += frames+ chunksPending += 1lock.unlock()+ let channelCount = Int(buffer.format.channelCount)+ var mono = [Float](repeating: 0, count: frames)+ for frame in 0..<frames {+ for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }+ mono[frame] /= Float(channelCount)+ }+ let samples = mono+ queue.async { [self] in+ defer {+ lock.lock(); framesPending -= frames; chunksPending -= 1; lock.unlock()+ }+ do {+ meterFrames += frames+ meterEnergy += samples.reduce(0.0) { $0 + Double($1 * $1) }+ if meterFrames >= rate / 10 {+ let rms = sqrt(meterEnergy / Double(meterFrames))+ level(Float(min(1, max(0, (20 * log10(max(rms, 0.00001)) + 55) / 45))))+ meterFrames = 0; meterEnergy = 0+ }+ for start in stride(from: 0, to: frames, by: rate / 10) {+ let end = min(frames, start + rate / 10)+ var pcm = Data(count: (end - start) * 2)+ pcm.withUnsafeMutableBytes { raw in+ let values = raw.bindMemory(to: Int16.self)+ for index in start..<end {+ let value = samples[index]+ values[index - start] =+ Int16(min(max(value.isFinite ? value : 0, -1), 1) * 32767)+ .littleEndian+ }+ }+ try stream.feed(pcmLe: pcm)+ }+ } catch {+ lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()+ if wasAccepting { try? stream.abandon(reason: .failure); interrupted() }+ }+ }}}diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex 2c308bd5e7..aa73e87cd8 100644--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -118,7 +118,6 @@ actor LlmProvider {private var currentContextLength: Int?private var backendInitialized = falseprivate var chatWarmupOwner: UUID?- private var voiceOwner: UUID?private var chatForeground = falseprivate var appForeground = falseprivate var memoryPressure = ModelMemoryPressureState()@@ -169,51 +168,38 @@ actor LlmProvider {appForeground: appForeground, now: ProcessInfo.processInfo.systemUptime)else { return }- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()unloadModel()memoryPressure.didCleanUp()}- func prewarmVoice(owner: UUID) async throws {- try await withModelLock {- try Task.checkCancellation()- guard chatForeground, !memoryPressureSuppressed else { return }- try prepareVoice()- do {- let transcriber = transcriber- try await Task.detached { try transcriber.loadModel() }.value- try Task.checkCancellation()- voiceOwner = owner- } catch {- unloadTranscriptionModelIfLoaded()- throw error+ func runLiveVoice(+ stream: DictationStream, callback: any DictationStreamCallback+ ) async throws {+ do {+ try await withTaskCancellationHandler {+ try await withModelLock {+ try Task.checkCancellation()+ try prepareVoice()+ let transcriber = transcriber+ defer { transcriber.unloadModel() }+ try await Task.detached(priority: .userInitiated) {+ try stream.run(transcriber: transcriber, callback: callback)+ }.value+ try Task.checkCancellation()+ }+ } onCancel: {+ try? stream.abandon(reason: .interruption)}- }- }-- func releaseVoice(owner: UUID) async {- try? await modelLoadGate.withLock {- if voiceOwner == owner { unloadTranscriptionModelIfLoaded() }- }- }-- func transcribe(inputSampleRate: UInt32, pcmLe: Data) async throws -> String {- try await withModelLock {- try Task.checkCancellation()- try prepareVoice()- defer { unloadTranscriptionModelIfLoaded() }- let transcriber = transcriber- let result = try await Task.detached {- try transcriber.transcribe(inputSampleRate: inputSampleRate, pcmLe: pcmLe)- }.value- try Task.checkCancellation()- return result+ } catch {+ try? stream.abandon(reason: .failure)+ throw error}}private func prepareVoice() throws {loadedContext?.releaseMultimodal()- if voiceOwner == nil && !canAllocate(.voice) {+ if !canAllocate(.voice) {unloadModel()guard hasRequiredWorkReserve() else { throw ModelMemoryUnavailable() }}@@ -355,14 +341,14 @@ actor LlmProvider {_ selection: LlmModelSelection, owner: UUID) async throws -> ChatWarmupResult {let result: ChatWarmupResult = try await withModelLock(retainAfterCooldown: true) {- guard appForeground, chatForeground, voiceOwner == nil,- isChatModelReady(selection)- else { return .skipped }+ guard appForeground, chatForeground, isChatModelReady(selection) else {+ return .skipped+ }let cooldown = memoryPressure.remainingCooldown(now: ProcessInfo.processInfo.systemUptime)if cooldown > 0 { return .retryAfter(cooldown) }guard !memoryPressureSuppressed else { return .skipped }- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()try Task.checkCancellation()guard loadedContextLength(selection) == nil else {if chatWarmupOwner != nil { chatWarmupOwner = owner }@@ -410,7 +396,7 @@ actor LlmProvider {return}- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()unloadModel()if !backendInitialized {@@ -467,11 +453,11 @@ actor LlmProvider {try await withTaskCancellationHandler {try await withModelLock {try control.checkCancellation()- guard chatForeground, !memoryPressureSuppressed, voiceOwner == nil,+ guard chatForeground, !memoryPressureSuppressed,loadedContextLength(selection) != nil,let model = loadedModel, let loadedContextelse { throw ModelMemoryDeferred() }- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()loadedContext.releaseMultimodal()guard canAllocate(.title) else { throw ModelMemoryDeferred() }try control.checkCancellation()@@ -522,7 +508,7 @@ actor LlmProvider {guard let context = loadedContext else {throw CancellationError()}- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()try control.checkCancellation()return try await operation(context, control)}@@ -649,10 +635,10 @@ actor LlmProvider {throw EmbeddingAssetInvalidError()}- if maintenance && (voiceOwner != nil || memoryPressureSuppressed) {+ if maintenance && memoryPressureSuppressed {throw ModelMemoryDeferred()}- unloadTranscriptionModelIfLoaded()+ transcriber.unloadModel()loadedContext?.releaseMultimodal()if !canAllocate(.embedding, queryBytes: maintenance ? nil : queryBytes) {if maintenance && chatForeground && loadedModel != nil {@@ -736,11 +722,6 @@ actor LlmProvider {currentContextLength = nil}- private func unloadTranscriptionModelIfLoaded() {- voiceOwner = nil- transcriber.unloadModel()- }-private nonisolated func chatAsset(_ selection: LlmModelSelection) -> Asset {llmAsset(modelId: selection.id)}diff --git a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swiftindex 23b45c5b68..44292bdc0d 100644--- a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift+++ b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift@@ -26,6 +26,10 @@ enum ChatDeviceCapability: Equatable {}}+func supportsLiveVoice(totalMemoryBytes: UInt64 = ProcessInfo.processInfo.physicalMemory) -> Bool {+ totalMemoryBytes > 4 * 1024 * 1024 * 1024+}+func currentChatDeviceCapability() -> ChatDeviceCapability {chatDeviceCapability(totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)}diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swiftindex cc3ad618bd..2765c1b472 100644--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift@@ -20,6 +20,16 @@ private actor ModelOperationBarrier {@MainActorfinal class ModelMemoryCoordinationTests: XCTestCase {+ func testLiveVoiceRequiresMoreThanFourGiB() {+ let limit: UInt64 = 4 * 1024 * 1024 * 1024+ for bytes: UInt64 in [0, 4_000_000_000, limit] {+ XCTAssertFalse(supportsLiveVoice(totalMemoryBytes: bytes))+ }+ for bytes: UInt64 in [limit + 1, 8_000_000_000] {+ XCTAssertTrue(supportsLiveVoice(totalMemoryBytes: bytes))+ }+ }+func testMinimumRamBlocksOnlyKnownReadingsBelowTheFloor() {XCTAssertFalse(chatDeviceCapability(totalMemoryBytes: 1_599_999_999).isChatSupported)XCTAssertTrue(chatDeviceCapability(totalMemoryBytes: 1_600_000_000).isChatSupported)diff --git a/rust/bindings/uniffi/ensu/src/conversation.rs b/rust/bindings/uniffi/ensu/src/conversation.rsindex a13f7bd255..07a60c6368 100644--- a/rust/bindings/uniffi/ensu/src/conversation.rs+++ b/rust/bindings/uniffi/ensu/src/conversation.rs@@ -17,6 +17,10 @@ pub enum ConversationError {Cancelled,#[error("Conversation changed; retry the reply")]Stale,+ #[error(+ "This message is too long for the model, even without earlier messages. Shorten it or split it into smaller messages. Your full text is still available to edit or copy."+ )]+ CurrentInputTooLarge,#[error("{detail}")]Other { detail: String },}@@ -32,6 +36,7 @@ impl From<core::PrepareError> for ConversationError {match cause {core::PrepareError::Cancelled => Self::Cancelled,core::PrepareError::Stale => Self::Stale,+ core::PrepareError::CurrentInputTooLarge => Self::CurrentInputTooLarge,cause => error(cause),}}diff --git a/rust/bindings/uniffi/ensu/src/lib.rs b/rust/bindings/uniffi/ensu/src/lib.rsindex 917d5f5a46..b7386167fb 100644--- a/rust/bindings/uniffi/ensu/src/lib.rs+++ b/rust/bindings/uniffi/ensu/src/lib.rs@@ -11,3 +11,4 @@ pub mod log;pub mod notes;pub mod retrieval;pub mod transcription;+pub mod voice;diff --git a/rust/bindings/uniffi/ensu/src/transcription.rs b/rust/bindings/uniffi/ensu/src/transcription.rsindex f149f83b5b..5163b6f068 100644--- a/rust/bindings/uniffi/ensu/src/transcription.rs+++ b/rust/bindings/uniffi/ensu/src/transcription.rs@@ -41,7 +41,7 @@ impl From<transcription::TranscriptionError> for TranscriptionError {#[derive(uniffi::Object)]pub struct Transcriber {- inner: transcription::Transcriber,+ pub(crate) inner: transcription::Transcriber,}#[uniffi::export]@@ -53,6 +53,23 @@ impl Transcriber {})}+ #[uniffi::constructor]+ pub fn with_cache(+ model_dir: String,+ vad_model_path: String,+ cache_directory: String,+ device_identity: String,+ ) -> Arc<Self> {+ Arc::new(Self {+ inner: transcription::Transcriber::with_cache(+ model_dir,+ vad_model_path,+ cache_directory,+ device_identity,+ ),+ })+ }+pub fn load_model(&self) -> Result<(), TranscriptionError> {self.inner.load_model().map_err(Into::into)}diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rsnew file mode 100644index 0000000000..ba3f5cb152--- /dev/null+++ b/rust/bindings/uniffi/ensu/src/voice.rs@@ -0,0 +1,497 @@+use ente_ensu::transcription::{DictationState, LiveControl, LiveEvent, LiveTranscription};+use ente_ensu::voice;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};+use std::time::{Duration, Instant};++#[derive(Debug, thiserror::Error, uniffi::Error)]+pub enum DictationError {+ #[error("{detail}")]+ InvalidOperation { detail: String },+}++impl From<voice::VoiceError> for DictationError {+ fn from(value: voice::VoiceError) -> Self {+ Self::InvalidOperation {+ detail: value.to_string(),+ }+ }+}++#[derive(Debug, Clone, uniffi::Record)]+pub struct DictationInsertionResult {+ pub text: String,+ pub revision: u64,+ pub caret_utf16: u32,+}++#[derive(uniffi::Object)]+pub struct DictationInsertion {+ anchor: voice::ComposerAnchor,+ transfer: Mutex<voice::ComposerTransfer>,+}++#[uniffi::export]+impl DictationInsertion {+ #[uniffi::constructor]+ pub fn new(destination: String, revision: u64, start_utf16: u32, end_utf16: u32) -> Arc<Self> {+ Arc::new(Self {+ anchor: voice::ComposerAnchor {+ destination,+ revision,+ start_utf16,+ end_utf16,+ },+ transfer: Mutex::new(voice::ComposerTransfer::default()),+ })+ }++ pub fn preview(+ &self,+ text: String,+ voice_text: String,+ ) -> Result<DictationInsertionResult, DictationError> {+ let (text, caret_utf16) = voice::preview_insertion(&text, &self.anchor, &voice_text)?;+ Ok(DictationInsertionResult {+ text,+ revision: self.anchor.revision,+ caret_utf16,+ })+ }++ pub fn apply(+ &self,+ destination: String,+ mut revision: u64,+ mut text: String,+ voice_text: String,+ foreground: bool,+ keyboard_composing: bool,+ ) -> Result<DictationInsertionResult, DictationError> {+ let caret_utf16 = self+ .transfer+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .apply(+ &voice::VoiceInsertion {+ anchor: self.anchor.clone(),+ text: voice_text,+ },+ &destination,+ &mut revision,+ &mut text,+ foreground,+ keyboard_composing,+ )?;+ Ok(DictationInsertionResult {+ text,+ revision,+ caret_utf16,+ })+ }+}++#[derive(Debug, Clone, Copy, uniffi::Enum)]+pub enum DictationPhase {+ Preparing,+ Listening,+ Countdown,+ Finishing,+ Ready,+ Closed,+}++#[derive(Debug, Clone, Copy, uniffi::Enum)]+pub enum DictationStopReason {+ Done,+ Interruption,+ Failure,+ Discard,+}+impl From<DictationStopReason> for voice::StopReason {+ fn from(value: DictationStopReason) -> Self {+ match value {+ DictationStopReason::Done => Self::Done,+ DictationStopReason::Interruption => Self::Interruption,+ DictationStopReason::Failure => Self::Failure,+ DictationStopReason::Discard => Self::Discard,+ }+ }+}+#[derive(Debug, Clone, uniffi::Record)]+pub struct DictationSnapshot {+ pub phase: DictationPhase,+ pub text: String,+ pub hypothesis: String,+ pub worker_active: bool,+ pub incomplete: bool,+}+#[derive(uniffi::Object)]+pub struct DictationSession {+ state: Mutex<DictationState>,+ live: bool,+}++#[uniffi::export(foreign)]+pub trait DictationStreamCallback: Send + Sync {+ fn on_ready(&self);+ fn on_change(&self);+ fn on_capture_closed(&self);+}++#[derive(uniffi::Object)]+pub struct DictationStream {+ session: Arc<DictationSession>,+ stream: LiveTranscription,+ started: AtomicBool,+}+#[uniffi::export]+impl DictationStream {+ #[uniffi::constructor]+ pub fn new(+ session: Arc<DictationSession>,+ sample_rate: u32,+ channels: u16,+ ) -> Result<Arc<Self>, DictationError> {+ let stream = LiveTranscription::new(voice::CaptureFormat {+ sample_rate,+ channels,+ })?;+ session.lock().session.begin_capture()?;+ Ok(Arc::new(Self {+ session,+ stream,+ started: AtomicBool::new(false),+ }))+ }+ pub fn feed(&self, pcm_le: &[u8]) -> Result<(), DictationError> {+ self.stream.feed(pcm_le).map_err(Into::into)+ }+ pub fn stop(&self, reason: DictationStopReason) -> Result<(), DictationError> {+ self.session.lock().session.stop(reason.into())?;+ if matches!(+ reason,+ DictationStopReason::Discard+ | DictationStopReason::Interruption+ | DictationStopReason::Failure+ ) {+ self.stream.cancel();+ } else {+ self.stream.finish();+ }+ Ok(())+ }+ pub fn abandon(&self, reason: DictationStopReason) -> Result<(), DictationError> {+ if !matches!(+ reason,+ DictationStopReason::Failure+ | DictationStopReason::Interruption+ | DictationStopReason::Discard+ ) {+ return Err(voice::VoiceError::InvalidState.into());+ }+ self.stop(reason)?;+ if self+ .started+ .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)+ .is_ok()+ {+ self.session.lock().session.worker_exited(false)?;+ }+ Ok(())+ }+ pub fn run(+ &self,+ transcriber: Arc<crate::transcription::Transcriber>,+ callback: Arc<dyn DictationStreamCallback>,+ ) -> Result<(), DictationError> {+ if self.started.swap(true, Ordering::AcqRel) {+ return Err(voice::VoiceError::InvalidState.into());+ }+ let mut notified_stop = false;+ let mut last_notification = Instant::now();+ let mut on_event = |event: LiveEvent| {+ let immediate = matches!(+ &event,+ LiveEvent::Commit { .. }+ | LiveEvent::ReplaceTranscript(_)+ | LiveEvent::Deferred+ | LiveEvent::CaptureClosed+ );+ if matches!(&event, LiveEvent::Ready) {+ let phase = self.session.lock().session.phase();+ if phase == voice::VoicePhase::Closed {+ return Ok(LiveControl::Cancel);+ }+ if phase == voice::VoicePhase::Listening {+ callback.on_ready();+ }+ return Ok(LiveControl::Continue);+ }+ let control = self+ .session+ .lock()+ .apply(event, || self.stream.finish_if_empty())?;+ if immediate && !notified_stop && self.session.lock().session.stop_reason().is_some() {+ notified_stop = true;+ callback.on_capture_closed();+ }+ if immediate || last_notification.elapsed() >= Duration::from_millis(250) {+ callback.on_change();+ last_notification = Instant::now();+ }+ Ok(control)+ };+ let result = if self.session.live {+ self.stream.run(&transcriber.inner, &mut on_event)+ } else {+ self.stream.run_recorded(&transcriber.inner, &mut on_event)+ };+ let mut state = self.session.lock();+ if result.is_err() && state.session.phase() != voice::VoicePhase::Closed {+ if state.session.stop_reason() != Some(voice::StopReason::Interruption) {+ state.session.stop(voice::StopReason::Failure)?;+ }+ let accepted = state.session.accepted_samples();+ let submitted = self.stream.submitted_samples();+ if submitted > accepted {+ state.session.observe_drained_audio(accepted, submitted)?;+ }+ }+ state.session.worker_exited(result.is_ok())?;+ drop(state);+ self.stream.finish();+ if !notified_stop {+ callback.on_capture_closed();+ }+ callback.on_change();+ result.map_err(Into::into)+ }+}+impl DictationSession {+ fn lock(&self) -> MutexGuard<'_, DictationState> {+ self.state.lock().unwrap_or_else(PoisonError::into_inner)+ }+}+#[uniffi::export]+impl DictationSession {+ #[uniffi::constructor]+ pub fn new(live: bool) -> Arc<Self> {+ Arc::new(Self {+ live,+ state: Mutex::new(DictationState::new(live)),+ })+ }+ pub fn stop(&self, reason: DictationStopReason) -> Result<bool, DictationError> {+ self.lock().session.stop(reason.into()).map_err(Into::into)+ }+ pub fn snapshot(&self) -> DictationSnapshot {+ let state = self.lock();+ let phase = match state.session.phase() {+ voice::VoicePhase::Preparing => DictationPhase::Preparing,+ voice::VoicePhase::Listening => DictationPhase::Listening,+ voice::VoicePhase::Countdown => DictationPhase::Countdown,+ voice::VoicePhase::Finishing => DictationPhase::Finishing,+ voice::VoicePhase::Ready => DictationPhase::Ready,+ voice::VoicePhase::Closed => DictationPhase::Closed,+ };+ DictationSnapshot {+ phase,+ text: state.text().to_owned(),+ hypothesis: state.hypothesis(),+ worker_active: state.session.worker_active(),+ incomplete: state.session.incomplete(),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ #[derive(Default)]+ struct StreamSignals {+ stopped: std::sync::atomic::AtomicUsize,+ ready: std::sync::atomic::AtomicUsize,+ }+ impl DictationStreamCallback for StreamSignals {+ fn on_ready(&self) {+ self.ready.fetch_add(1, Ordering::Relaxed);+ }+ fn on_change(&self) {}+ fn on_capture_closed(&self) {+ self.stopped.fetch_add(1, Ordering::Relaxed);+ }+ }++ #[test]+ fn composer_previews_replace_the_selection_without_consuming_the_transfer() {+ let insertion = DictationInsertion::new("chat".into(), 7, 7, 13);+ let original = " meet Friday\n".to_owned();+ assert_eq!(+ insertion+ .preview(original.clone(), String::new())+ .unwrap()+ .text,+ original+ );+ assert_eq!(+ insertion+ .preview(original.clone(), "Sat".into())+ .unwrap()+ .text,+ " meet Sat\n"+ );+ let preview = insertion+ .preview(original.clone(), "Saturday".into())+ .unwrap();+ assert_eq!(preview.revision, 7);+ let final_text = insertion+ .apply("chat".into(), 7, original, "Saturday".into(), true, false)+ .unwrap();+ assert_eq!(preview.text, final_text.text);+ assert_eq!(preview.caret_utf16, final_text.caret_utf16);+ assert_eq!(final_text.revision, 8);+ assert!(+ insertion+ .apply(+ "chat".into(),+ 8,+ final_text.text,+ "Sunday".into(),+ true,+ false+ )+ .is_err()+ );+ }++ fn event(session: &DictationSession, event: LiveEvent) {+ session.lock().apply(event, || true).unwrap();+ }++ #[test]+ fn interruption_keeps_visible_text_until_actual_worker_exit() {+ let owner = DictationSession::new(true);+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();+ stream.started.store(true, Ordering::Release);+ event(+ &owner,+ LiveEvent::Audio {+ start: 0,+ end: 32000,+ voiced: true,+ },+ );+ event(+ &owner,+ LiveEvent::Commit {+ start: 0,+ end: 16000,+ text: "Keep this".into(),+ },+ );+ event(&owner, LiveEvent::Hypothesis("opening word".into()));+ stream.stop(DictationStopReason::Done).unwrap();+ stream.abandon(DictationStopReason::Interruption).unwrap();+ assert!(owner.snapshot().worker_active);+ assert!(DictationStream::new(owner.clone(), 16000, 1).is_err());+ owner.lock().session.worker_exited(false).unwrap();+ let value = owner.snapshot();+ assert_eq!(+ format!("{}{}", value.text, value.hypothesis),+ "Keep this opening word"+ );+ assert!(!value.worker_active);+ assert!(value.incomplete);+ assert!(matches!(value.phase, DictationPhase::Ready));+ let insertion = DictationInsertion::new("chat".into(), 0, 7, 7);+ let draft = insertion+ .apply(+ "chat".into(),+ 0,+ "Notes: ".into(),+ value.text + &value.hypothesis,+ true,+ false,+ )+ .unwrap();+ assert_eq!(draft.text, "Notes: Keep this opening word");+ assert!(matches!(+ owner+ .lock()+ .apply(LiveEvent::Hypothesis("late".into()), || true)+ .unwrap(),+ LiveControl::Cancel+ ));+ assert!(DictationStream::new(owner, 16000, 1).is_err());+ assert!(DictationStream::new(DictationSession::new(true), 16000, 1).is_ok());+ }+ #[test]+ fn format_failure_does_not_reserve_a_worker_and_cancel_before_run_cannot_resume() {+ let owner = DictationSession::new(true);+ assert!(DictationStream::new(owner.clone(), 0, 1).is_err());+ assert!(!owner.snapshot().worker_active);+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();+ stream.abandon(DictationStopReason::Failure).unwrap();+ assert!(!owner.snapshot().worker_active);+ assert!(DictationStream::new(owner, 16000, 1).is_err());+ assert!(stream.feed(&[0, 0]).is_err());+ }+ #[test]+ fn missing_models_close_capture_once_without_starting_hardware() {+ for live in [true, false] {+ let owner = DictationSession::new(live);+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();+ let signals = Arc::new(StreamSignals::default());+ assert!(+ stream+ .run(+ crate::transcription::Transcriber::new("missing".into(), "missing".into()),+ signals.clone()+ )+ .is_err()+ );+ assert_eq!(signals.ready.load(Ordering::Relaxed), 0);+ assert_eq!(signals.stopped.load(Ordering::Relaxed), 1);+ assert!(!owner.snapshot().worker_active);+ assert!(owner.snapshot().incomplete);+ }+ }+ #[test]+ fn replacement_matches_the_composer_preview_and_discard_rejects_late_text() {+ let owner = DictationSession::new(true);+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();+ event(+ &owner,+ LiveEvent::Commit {+ start: 0,+ end: 16000,+ text: "partial".into(),+ },+ );+ event(&owner, LiveEvent::Hypothesis("preview".into()));+ assert!(+ owner+ .lock()+ .apply(+ LiveEvent::Commit {+ start: 1,+ end: 16000,+ text: "overlap".into()+ },+ || true+ )+ .is_err()+ );+ event(+ &owner,+ LiveEvent::ReplaceTranscript("Complete recording".into()),+ );+ assert_eq!(owner.snapshot().text, "Complete recording");+ assert!(owner.snapshot().hypothesis.is_empty());+ stream.abandon(DictationStopReason::Discard).unwrap();+ event(&owner, LiveEvent::ReplaceTranscript("late".into()));+ assert_eq!(owner.snapshot().text, "Complete recording");+ assert!(matches!(owner.snapshot().phase, DictationPhase::Closed));+ }+}diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.jsonindex c9add6495a..9280db1850 100644--- a/rust/checks/lint-exceptions/suppressions.json+++ b/rust/checks/lint-exceptions/suppressions.json@@ -27,7 +27,6 @@"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],"rust/crates/ensu/src/platform.rs": ["unsafe_code"],- "rust/crates/ensu/src/transcription/text.rs": ["clippy::expect_used"],"rust/crates/exif/src/isobmff.rs": ["clippy::expect_used"],"rust/crates/image/src/color_management.rs": ["clippy::expect_used"],"rust/crates/location/src/binary.rs": ["clippy::expect_used"],diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex f2d16c6332..0a66e06b30 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -7,6 +7,7 @@ ente-assets.workspace = trueente-core.workspace = trueente-ensu-crypto.workspace = trueente-image.workspace = true+ort = { workspace = true, optional = true }regex.workspace = truerubato = { version = "5.0.0", optional = true }rusqlite = { workspace = true, default-features = true, optional = true }@@ -15,7 +16,7 @@ serde.workspace = trueserde_json.workspace = truethiserror.workspace = truetokio = { workspace = true, features = ["rt", "time"] }-transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }+transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx"], optional = true }url.workspace = trueuuid = { workspace = true, features = ["v4", "rng-getrandom"] }zeroize.workspace = truediff --git a/rust/crates/ensu/src/lib.rs b/rust/crates/ensu/src/lib.rsindex 9cfb5e7919..1f77467272 100644--- a/rust/crates/ensu/src/lib.rs+++ b/rust/crates/ensu/src/lib.rs@@ -12,3 +12,5 @@ mod platform;#[cfg(feature = "transcription")]pub mod transcription;+#[cfg(feature = "transcription")]+pub mod voice;diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rsindex f8bb38792d..0b3d1b9083 100644--- a/rust/crates/ensu/src/transcription/audio.rs+++ b/rust/crates/ensu/src/transcription/audio.rs@@ -3,13 +3,12 @@ use std::time::Duration;use rubato::audioadapter_buffers::direct::InterleavedSlice;use rubato::{Fft, FixedSync, Resampler, WindowFunction};-use transcribe_rs::vad::{SileroVad, SmoothedVad, Vad};+use transcribe_rs::vad::{SmoothedVad, Vad};-use crate::transcription::{Result, TranscriptionError};+use crate::transcription::{Result, SileroV4Vad, TranscriptionError};const TARGET_SAMPLE_RATE: usize = 16_000;const RESAMPLER_CHUNK_SIZE: usize = 1024;-const VAD_THRESHOLD: f32 = 0.5;const VAD_PREFILL_FRAMES: usize = 3;const VAD_HANGOVER_FRAMES: usize = 10;const VAD_ONSET_FRAMES: usize = 2;@@ -53,7 +52,7 @@ fn extract_speech(if !vad_model_path.is_file() {return Err(TranscriptionError::VadNotDownloaded);}- let silero = SileroVad::new(vad_model_path, VAD_THRESHOLD)?;+ let silero = SileroV4Vad::new(vad_model_path)?;let mut vad = SmoothedVad::new(Box::new(silero),VAD_PREFILL_FRAMES,@@ -61,31 +60,39 @@ fn extract_speech(VAD_ONSET_FRAMES,);+ extract_speech_with_vad(input_sample_rate, samples, &mut vad)+}++fn extract_speech_with_vad(+ input_sample_rate: u32,+ samples: &[f32],+ vad: &mut dyn Vad,+) -> Result<Vec<f32>> {let mut resampler = FrameResampler::new(input_sample_rate as usize,TARGET_SAMPLE_RATE,- Duration::from_millis(30),+ Duration::from_secs_f64(vad.frame_size() as f64 / TARGET_SAMPLE_RATE as f64),)?;let mut speech = Vec::new();-- resampler.push(samples, |frame| {- if let Ok(true) = vad.is_speech(frame) {- let prefill = vad.drain_prefill();- if !prefill.is_empty() {- speech.extend_from_slice(&prefill);- }- speech.extend_from_slice(frame);+ let mut vad_error = None;+ let mut emit = |frame: &[f32]| {+ if vad_error.is_some() {+ return;}- });- resampler.finish(|frame| {- if let Ok(true) = vad.is_speech(frame) {- let prefill = vad.drain_prefill();- if !prefill.is_empty() {- speech.extend_from_slice(&prefill);+ match vad.is_speech(frame) {+ Ok(true) => {+ speech.extend_from_slice(&vad.drain_prefill());+ speech.extend_from_slice(frame);}- speech.extend_from_slice(frame);+ Ok(false) => {}+ Err(error) => vad_error = Some(error),}- });+ };+ resampler.push(samples, &mut emit);+ resampler.finish(&mut emit);+ if let Some(error) = vad_error {+ return Err(error.into());+ }vad.reset();let min_samples = TARGET_SAMPLE_RATE * 5 / 4;@@ -200,6 +207,35 @@ mod tests {use super::*;+ #[test]+ fn vad_failure_does_not_return_an_apparently_complete_partial_recording() {+ struct FailsAfterSpeech {+ frames: usize,+ }+ impl Vad for FailsAfterSpeech {+ fn frame_size(&self) -> usize {+ 480+ }+ fn is_speech(+ &mut self,+ _: &[f32],+ ) -> std::result::Result<bool, transcribe_rs::TranscribeError> {+ self.frames += 1;+ if self.frames == 1 {+ Ok(true)+ } else {+ Err(transcribe_rs::TranscribeError::Inference(+ "injected VAD failure".into(),+ ))+ }+ }+ fn reset(&mut self) {}+ }+ let mut vad = FailsAfterSpeech { frames: 0 };+ assert!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad).is_err());+ assert_eq!(vad.frames, 2);+ }+fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {let mut resampler =FrameResampler::new(sample_rate, 16_000, Duration::from_millis(30)).unwrap();diff --git a/rust/crates/ensu/src/transcription/graph_cache.rs b/rust/crates/ensu/src/transcription/graph_cache.rsnew file mode 100644index 0000000000..807f739cc6--- /dev/null+++ b/rust/crates/ensu/src/transcription/graph_cache.rs@@ -0,0 +1,651 @@+use std::fs::{self, File, OpenOptions};+use std::io::{self, Read, Write};+use std::path::{Path, PathBuf};+use std::sync::Mutex;+use std::time::UNIX_EPOCH;++use ente_ensu_crypto::{Sha256Hasher, sha256};+use serde::{Deserialize, Serialize};+use uuid::Uuid;++pub(super) const GRAPHS: [&str; 3] = [+ "encoder-model.int8.onnx",+ "decoder_joint-model.int8.onnx",+ "nemo128.onnx",+];+pub(super) const SOURCES: [&str; 4] = [GRAPHS[0], GRAPHS[1], GRAPHS[2], "vocab.txt"];+const SCHEMA: u32 = 2;+const CACHE_DIRECTORY: &str = "ensu-voice-onnx-v1";+const MAX_BYTES: u64 = 1536 * 1024 * 1024;+const MAX_MANIFEST: u64 = 64 * 1024;++pub(super) enum LoadPlan<'a> {+ Source { save_to: Option<&'a Path> },+ Cached { directory: &'a Path },+}++pub(super) struct GraphCache {+ root: PathBuf,+ device: String,+ verified: Mutex<Option<Verified>>,+}++#[derive(PartialEq, Eq)]+struct Verified {+ recipe: String,+ sources: Vec<SourceStamp>,+ files: Vec<SourceStamp>,+}++#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]+struct SourceStamp {+ bytes: u64,+ modified_ns: u128,+ #[cfg(unix)]+ device: u64,+ #[cfg(unix)]+ inode: u64,+ #[cfg(unix)]+ changed_secs: i64,+ #[cfg(unix)]+ changed_ns: i64,+}++#[derive(Serialize, Deserialize, PartialEq, Eq)]+struct Artifact {+ bytes: u64,+ sha256: String,+}++#[derive(Serialize, Deserialize)]+struct Manifest {+ schema: u32,+ recipe: String,+ source_stamps: Vec<SourceStamp>,+ #[cfg(not(unix))]+ source_sha256: Vec<String>,+ artifacts: Vec<Artifact>,+}++struct Staging(PathBuf);+impl Drop for Staging {+ fn drop(&mut self) {+ let _ = remove_owned_directory(&self.0);+ }+}++impl GraphCache {+ pub fn new(cache_directory: PathBuf, device_identity: String) -> Self {+ Self {+ root: cache_directory.join(CACHE_DIRECTORY),+ device: device_identity,+ verified: Mutex::new(None),+ }+ }++ pub fn load<T, E>(+ &self,+ model_dir: &Path,+ runtime_configuration: &str,+ may_build: bool,+ mut loader: impl FnMut(LoadPlan<'_>) -> Result<T, E>,+ ) -> Result<T, E> {+ let Some((_lock, stamps)) = self.lock_and_stat(model_dir) else {+ return loader(LoadPlan::Source { save_to: None });+ };+ let recipe = digest_bytes(+ format!(+ "schema={SCHEMA};upstream=efc66111;ort-api={};os={};arch={};device={};{runtime_configuration}",+ ort::MINOR_VERSION,+ std::env::consts::OS,+ std::env::consts::ARCH,+ self.device,+ )+ .as_bytes(),+ );+ let ready = self.root.join("ready");+ let fingerprint = cache_fingerprint(&ready, &recipe, &stamps).ok();+ let already_verified = cfg!(unix)+ && fingerprint.as_ref().is_some_and(|fingerprint| {+ self.verified+ .lock()+ .ok()+ .is_some_and(|v| v.as_ref() == Some(fingerprint))+ });+ if validate(&ready, model_dir, &recipe, &stamps, already_verified).is_ok()+ && let Ok(model) = loader(LoadPlan::Cached { directory: &ready })+ && source_stamps(model_dir).ok().as_ref() == Some(&stamps)+ && cache_fingerprint(&ready, &recipe, &stamps).ok() == fingerprint+ {+ if let Ok(mut verified) = self.verified.lock() {+ *verified = fingerprint;+ }+ return Ok(model);+ }+ if let Ok(mut verified) = self.verified.lock() {+ *verified = None;+ }+ if !may_build || self.clean_for_build().is_err() {+ return loader(LoadPlan::Source { save_to: None });+ }+ let staging = Staging(self.root.join(format!(".build-{}", Uuid::new_v4())));+ if fs::create_dir(&staging.0).is_err() {+ return loader(LoadPlan::Source { save_to: None });+ }+ if stamps.last().is_none_or(|stamp| stamp.bytes > MAX_BYTES)+ || fs::copy(model_dir.join("vocab.txt"), staging.0.join("vocab.txt")).is_err()+ {+ return loader(LoadPlan::Source { save_to: None });+ }+ #[cfg(not(unix))]+ let Ok(source_sha256) = source_hashes(model_dir) else {+ return loader(LoadPlan::Source { save_to: None });+ };+ let Ok(model) = loader(LoadPlan::Source {+ save_to: Some(&staging.0),+ }) else {+ drop(staging);+ return loader(LoadPlan::Source { save_to: None });+ };+ let publish = (|| -> io::Result<()> {+ if source_stamps(model_dir)? != stamps {+ return Err(invalid_cache());+ }+ let artifacts = artifacts(&staging.0)?;+ let manifest = Manifest {+ schema: SCHEMA,+ recipe: recipe.clone(),+ source_stamps: stamps.clone(),+ #[cfg(not(unix))]+ source_sha256,+ artifacts,+ };+ let mut file = OpenOptions::new()+ .write(true)+ .create_new(true)+ .open(staging.0.join("manifest.json"))?;+ file.write_all(&serde_json::to_vec(&manifest)?)?;+ file.sync_all()?;+ fs::rename(&staging.0, &ready)?;+ Ok(())+ })();+ if publish.is_ok()+ && let Ok(mut verified) = self.verified.lock()+ {+ *verified = cache_fingerprint(&ready, &recipe, &stamps).ok();+ }+ Ok(model)+ }++ fn lock_and_stat(&self, model_dir: &Path) -> Option<(File, Vec<SourceStamp>)> {+ fs::create_dir_all(&self.root).ok()?;+ if !fs::symlink_metadata(&self.root).ok()?.is_dir() {+ return None;+ }+ let lock = OpenOptions::new()+ .create(true)+ .truncate(false)+ .read(true)+ .write(true)+ .open(self.root.join(".lock"))+ .ok()?;+ lock.try_lock().ok()?;+ Some((lock, source_stamps(model_dir).ok()?))+ }++ fn clean_for_build(&self) -> io::Result<()> {+ remove_owned_directory(&self.root.join("ready"))?;+ for entry in fs::read_dir(&self.root)? {+ let entry = entry?;+ let name = entry.file_name();+ if name+ .to_str()+ .and_then(|s| s.strip_prefix(".build-"))+ .and_then(|s| Uuid::parse_str(s).ok())+ .is_some()+ {+ remove_owned_directory(&entry.path())?;+ }+ }+ Ok(())+ }+}++fn source_stamps(model_dir: &Path) -> io::Result<Vec<SourceStamp>> {+ stamps_for(model_dir, &SOURCES)+}++fn cache_fingerprint(ready: &Path, recipe: &str, sources: &[SourceStamp]) -> io::Result<Verified> {+ Ok(Verified {+ recipe: recipe.into(),+ sources: sources.to_vec(),+ files: stamps_for(+ ready,+ &[+ GRAPHS[0],+ GRAPHS[1],+ GRAPHS[2],+ "vocab.txt",+ "manifest.json",+ ],+ )?,+ })+}++fn stamps_for(directory: &Path, names: &[&str]) -> io::Result<Vec<SourceStamp>> {+ names+ .iter()+ .map(|name| {+ let metadata = fs::symlink_metadata(directory.join(name))?;+ if !metadata.is_file() || metadata.len() == 0 {+ return Err(invalid_cache());+ }+ let modified_ns = metadata+ .modified()?+ .duration_since(UNIX_EPOCH)+ .map_err(|_| invalid_cache())?+ .as_nanos();+ #[cfg(unix)]+ use std::os::unix::fs::MetadataExt;+ Ok(SourceStamp {+ bytes: metadata.len(),+ modified_ns,+ #[cfg(unix)]+ device: metadata.dev(),+ #[cfg(unix)]+ inode: metadata.ino(),+ #[cfg(unix)]+ changed_secs: metadata.ctime(),+ #[cfg(unix)]+ changed_ns: metadata.ctime_nsec(),+ })+ })+ .collect()+}++#[cfg(not(unix))]+fn source_hashes(model_dir: &Path) -> io::Result<Vec<String>> {+ SOURCES+ .iter()+ .map(|name| digest_file(&model_dir.join(name)).map(|a| a.sha256))+ .collect()+}++fn validate(+ ready: &Path,+ model_dir: &Path,+ recipe: &str,+ stamps: &[SourceStamp],+ already_verified: bool,+) -> io::Result<()> {+ if !fs::symlink_metadata(ready)?.is_dir() {+ return Err(invalid_cache());+ }+ let path = ready.join("manifest.json");+ let metadata = fs::symlink_metadata(&path)?;+ if !metadata.is_file() || metadata.len() > MAX_MANIFEST {+ return Err(invalid_cache());+ }+ let mut bytes = Vec::new();+ File::open(path)?+ .take(MAX_MANIFEST + 1)+ .read_to_end(&mut bytes)?;+ if bytes.len() as u64 > MAX_MANIFEST {+ return Err(invalid_cache());+ }+ let manifest: Manifest = serde_json::from_slice(&bytes)?;+ if manifest.schema != SCHEMA+ || manifest.recipe != recipe+ || manifest.source_stamps != stamps+ || manifest.artifacts.len() != SOURCES.len()+ {+ return Err(invalid_cache());+ }+ #[cfg(not(unix))]+ if source_hashes(model_dir)? != manifest.source_sha256 {+ return Err(invalid_cache());+ }+ if already_verified {+ validate_directory_entries(ready)?;+ let bytes: u64 = manifest.artifacts.iter().map(|g| g.bytes).sum();+ if bytes > MAX_BYTES {+ return Err(invalid_cache());+ }+ return Ok(());+ }+ if artifacts(ready)? != manifest.artifacts || source_stamps(model_dir)? != stamps {+ return Err(invalid_cache());+ }+ Ok(())+}++fn artifacts(directory: &Path) -> io::Result<Vec<Artifact>> {+ let mut total = 0;+ let mut artifacts = Vec::new();+ for name in SOURCES {+ let artifact = digest_file(&directory.join(name))?;+ total += artifact.bytes;+ if total > MAX_BYTES {+ return Err(invalid_cache());+ }+ artifacts.push(artifact);+ }+ validate_directory_entries(directory)?;+ Ok(artifacts)+}++fn validate_directory_entries(directory: &Path) -> io::Result<()> {+ for entry in fs::read_dir(directory)? {+ let name = entry?.file_name();+ if !SOURCES.iter().any(|expected| name == *expected) && name != "manifest.json" {+ return Err(invalid_cache());+ }+ }+ Ok(())+}++fn digest_file(path: &Path) -> io::Result<Artifact> {+ let metadata = fs::symlink_metadata(path)?;+ if !metadata.is_file() || metadata.len() == 0 || metadata.len() > MAX_BYTES {+ return Err(invalid_cache());+ }+ let mut file = File::open(path)?;+ let mut hasher = Sha256Hasher::new();+ let mut buffer = vec![0_u8; 64 * 1024];+ let mut bytes = 0;+ loop {+ let read = file.read(&mut buffer)?;+ if read == 0 {+ break;+ }+ bytes += read as u64;+ if bytes > MAX_BYTES {+ return Err(invalid_cache());+ }+ hasher.update(&buffer[..read]);+ }+ if bytes != metadata.len() {+ return Err(invalid_cache());+ }+ Ok(Artifact {+ bytes,+ sha256: hex(&hasher.finalize()),+ })+}++fn digest_bytes(bytes: &[u8]) -> String {+ hex(&sha256(bytes))+}++fn hex(bytes: &[u8]) -> String {+ bytes.iter().map(|b| format!("{b:02x}")).collect()+}++fn invalid_cache() -> io::Error {+ io::Error::new(io::ErrorKind::InvalidData, "Invalid voice graph cache")+}+fn remove_owned_directory(path: &Path) -> io::Result<()> {+ match fs::symlink_metadata(path) {+ Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path),+ Ok(_) => fs::remove_file(path),+ Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),+ Err(error) => Err(error),+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::cell::RefCell;++ struct Fixture {+ temp: tempfile::TempDir,+ source: PathBuf,+ cache: GraphCache,+ loads: RefCell<Vec<&'static str>>,+ }++ impl Fixture {+ fn new() -> Self {+ let temp = tempfile::tempdir().unwrap();+ let source = temp.path().join("source");+ fs::create_dir(&source).unwrap();+ for name in SOURCES {+ fs::write(source.join(name), name.as_bytes()).unwrap();+ }+ let cache = GraphCache::new(temp.path().join("cache"), "phone-a".into());+ Self {+ temp,+ source,+ cache,+ loads: RefCell::new(Vec::new()),+ }+ }++ fn load(&self, recipe: &str) -> String {+ self.cache+ .load(&self.source, recipe, true, |plan| self.loader(plan))+ .unwrap()+ }++ fn loader(&self, plan: LoadPlan<'_>) -> io::Result<String> {+ let (directory, save_to) = match plan {+ LoadPlan::Source { save_to } => {+ self.loads.borrow_mut().push(if save_to.is_some() {+ "build"+ } else {+ "source"+ });+ (self.source.as_path(), save_to)+ }+ LoadPlan::Cached { directory } => {+ self.loads.borrow_mut().push("hit");+ (directory, None)+ }+ };+ let mut result = String::new();+ for name in GRAPHS {+ let bytes = fs::read(directory.join(name))?;+ if let Some(save_to) = save_to {+ fs::write(save_to.join(name), &bytes)?;+ }+ result.push_str(&digest_bytes(&bytes));+ }+ Ok(result)+ }++ fn ready(&self) -> PathBuf {+ self.cache.root.join("ready")+ }+ }++ #[test]+ fn deferred_load_reuses_but_never_builds_graphs() {+ let f = Fixture::new();+ let expected = f+ .cache+ .load(&f.source, "runtime-a", false, |plan| f.loader(plan))+ .unwrap();+ assert!(!f.ready().exists());+ assert_eq!(f.load("runtime-a"), expected);+ assert_eq!(f.load("runtime-a"), expected);+ assert!(f.cache.verified.lock().unwrap().is_some());+ let restarted = GraphCache::new(f.temp.path().join("cache"), "phone-a".into());+ assert_eq!(+ restarted+ .load(&f.source, "runtime-a", false, |plan| f.loader(plan))+ .unwrap(),+ expected+ );+ assert!(restarted.verified.lock().unwrap().is_some());+ assert_eq!(f.loads.take(), ["source", "build", "hit", "hit"]);+ }++ #[test]+ fn changed_graph_source_runtime_device_and_manifest_rebuild() {+ let mut f = Fixture::new();+ let first = f.load("runtime-a;cpu");+ f.loads.take();+ for name in [GRAPHS[0], "vocab.txt"] {+ let path = f.ready().join(name);+ let mut bytes = fs::read(&path).unwrap();+ bytes[0] ^= 1;+ fs::write(&path, bytes).unwrap();+ assert_eq!(f.load("runtime-a;cpu"), first);+ assert_eq!(f.loads.take(), ["build"]);+ assert_eq!(+ fs::read(path).unwrap(),+ fs::read(f.source.join(name)).unwrap()+ );+ }++ let source = f.source.join(GRAPHS[0]);+ let mtime = fs::metadata(&source).unwrap().modified().unwrap();+ let mut bytes = fs::read(&source).unwrap();+ bytes[0] ^= 1;+ fs::write(&source, bytes).unwrap();+ File::options()+ .write(true)+ .open(source)+ .unwrap()+ .set_times(fs::FileTimes::new().set_modified(mtime))+ .unwrap();+ let updated = f.load("runtime-a;cpu");+ assert_ne!(updated, first);+ assert_eq!(f.loads.take(), ["build"]);+ for recipe in ["runtime-b;cpu", "runtime-b;xnnpack"] {+ assert_eq!(f.load(recipe), updated);+ assert_eq!(f.loads.take(), ["build"]);+ }+ f.cache.device = "phone-b".into();+ assert_eq!(f.load("runtime-b;xnnpack"), updated);+ assert_eq!(f.loads.take(), ["build"]);+ fs::write(f.ready().join("manifest.json"), b"{partial").unwrap();+ let abandoned = f.cache.root.join(format!(".build-{}", Uuid::new_v4()));+ fs::create_dir(&abandoned).unwrap();+ fs::write(abandoned.join("partial"), b"incomplete").unwrap();+ fs::write(f.cache.root.join("unrelated"), b"keep").unwrap();+ assert_eq!(f.load("runtime-b;xnnpack"), updated);+ assert_eq!(f.loads.take(), ["build"]);+ assert!(!abandoned.exists());+ assert_eq!(fs::read(f.cache.root.join("unrelated")).unwrap(), b"keep");+ assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 3);+ }++ #[test]+ fn missing_original_never_loads_a_surviving_cache() {+ let f = Fixture::new();+ f.load("runtime-a");+ fs::remove_file(f.source.join(GRAPHS[0])).unwrap();+ assert!(+ f.cache+ .load(&f.source, "runtime-a", true, |plan| f.loader(plan))+ .is_err()+ );+ assert_eq!(f.loads.take(), ["build", "source"]);+ }++ #[test]+ fn serializer_failure_retries_original_and_cleans_staging() {+ let f = Fixture::new();+ let result = f.cache.load(&f.source, "runtime-a", true, |plan| {+ if matches!(plan, LoadPlan::Source { save_to: Some(_) }) {+ f.loads.borrow_mut().push("failed-save");+ Err(io::Error::new(+ io::ErrorKind::StorageFull,+ "injected full disk",+ ))+ } else {+ f.loader(plan)+ }+ });+ assert!(result.is_ok());+ assert_eq!(f.loads.take(), ["failed-save", "source"]);+ assert!(!f.ready().exists());+ assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 1);+ }++ #[test]+ fn rejected_hash_valid_graph_rebuilds_from_source() {+ let f = Fixture::new();+ let expected = f.load("runtime-a");+ let actual = f+ .cache+ .load(&f.source, "runtime-a", true, |plan| {+ if matches!(plan, LoadPlan::Cached { .. }) {+ Err(invalid_cache())+ } else {+ f.loader(plan)+ }+ })+ .unwrap();+ assert_eq!(actual, expected);+ assert_eq!(f.loads.take(), ["build", "build"]);+ }++ #[test]+ fn busy_or_unusable_cache_never_blocks_source_loading() {+ let f = Fixture::new();+ let expected = f.load("runtime-a");+ let lock = File::options()+ .read(true)+ .write(true)+ .open(f.cache.root.join(".lock"))+ .unwrap();+ lock.try_lock().unwrap();+ assert_eq!(f.load("runtime-a"), expected);+ drop(lock);+ let file = f.temp.path().join("not-a-directory");+ fs::write(&file, b"existing").unwrap();+ let cache = GraphCache::new(file, "phone-a".into());+ assert_eq!(+ cache+ .load(&f.source, "runtime-a", true, |plan| f.loader(plan))+ .unwrap(),+ expected+ );+ assert_eq!(f.loads.take(), ["build", "source", "source"]);+ }++ #[test]+ fn oversized_or_external_output_is_not_published() {+ let f = Fixture::new();+ for oversize in [true, false] {+ let result = f.cache.load(&f.source, "runtime-a", true, |plan| {+ let save_to = match &plan {+ LoadPlan::Source { save_to } => *save_to,+ _ => None,+ };+ let result = f.loader(plan)?;+ if let Some(path) = save_to {+ if oversize {+ File::options()+ .write(true)+ .open(path.join(GRAPHS[0]))?+ .set_len(MAX_BYTES + 1)?;+ } else {+ fs::write(path.join("unverified.weights"), b"external")?;+ }+ }+ Ok::<_, io::Error>(result)+ });+ assert!(result.is_ok());+ assert!(!f.ready().exists());+ }+ }++ #[cfg(unix)]+ #[test]+ fn ready_symlink_cannot_redirect_loading_or_cleanup() {+ let f = Fixture::new();+ let expected = f.load("runtime-a");+ let outside = f.temp.path().join("outside");+ fs::create_dir(&outside).unwrap();+ fs::write(outside.join("preserve"), b"original").unwrap();+ fs::remove_dir_all(f.ready()).unwrap();+ std::os::unix::fs::symlink(&outside, f.ready()).unwrap();+ assert_eq!(f.load("runtime-a"), expected);+ assert_eq!(f.loads.take(), ["build", "build"]);+ assert_eq!(fs::read(outside.join("preserve")).unwrap(), b"original");+ }+}diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rsnew file mode 100644index 0000000000..33960e7154--- /dev/null+++ b/rust/crates/ensu/src/transcription/live.rs@@ -0,0 +1,642 @@+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::mpsc::{self, TryRecvError};+use std::time::Duration;++use transcribe_rs::vad::Vad;++use super::{StreamNormalizer, Transcriber};+use crate::voice::{+ AlignedWord, CaptureFormat, CaptureQueue, Result, RollingRecognizer, VoiceError,+};++pub enum LiveEvent {+ Ready,+ Audio { start: u64, end: u64, voiced: bool },+ Hypothesis(String),+ Commit { start: u64, end: u64, text: String },+ Deferred,+ ReplaceTranscript(String),+ Coverage(u64),+ Idle,+ CaptureClosed,+}++#[derive(Clone, Copy, PartialEq, Eq)]+pub enum LiveControl {+ Continue,+ Finish,+ Cancel,+}++enum DecoderEvent {+ Prepared(Result<()>),+ Window(crate::voice::DecodeWindow, Result<Vec<AlignedWord>>),+}++pub struct LiveTranscription {+ pub(super) queue: CaptureQueue,+ started: AtomicBool,+}++pub(super) struct CaptureGuard<'a>(&'a CaptureQueue);+impl Drop for CaptureGuard<'_> {+ fn drop(&mut self) {+ self.0.clear();+ }+}++impl LiveTranscription {+ pub fn new(format: CaptureFormat) -> Result<Self> {+ Ok(Self {+ queue: CaptureQueue::new(format)?,+ started: AtomicBool::new(false),+ })+ }++ pub fn feed(&self, pcm: &[u8]) -> Result<()> {+ self.queue.push(pcm)+ }+ pub fn finish(&self) {+ self.queue.finish();+ }+ pub fn finish_if_empty(&self) -> bool {+ self.queue.finish_if_empty()+ }+ pub fn cancel(&self) {+ self.queue.cancel();+ }+ pub fn submitted_samples(&self) -> u64 {+ self.queue.submitted_frames() * 16000 / u64::from(self.queue.format().sample_rate)+ }++ pub(super) fn start(&self) -> Result<CaptureGuard<'_>> {+ if self.started.swap(true, Ordering::AcqRel) {+ return Err(VoiceError::InvalidState);+ }+ let guard = CaptureGuard(&self.queue);+ self.check()?;+ Ok(guard)+ }++ pub(super) fn check(&self) -> Result<()> {+ if self.queue.is_cancelled() {+ Err(VoiceError::Cancelled)+ } else if self.queue.failed() {+ Err(VoiceError::BacklogFull)+ } else {+ Ok(())+ }+ }++ pub(super) fn emit(+ &self,+ event: LiveEvent,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ let idle = matches!(event, LiveEvent::Idle);+ match on_event(event)? {+ LiveControl::Cancel => {+ self.cancel();+ return Err(VoiceError::Cancelled);+ }+ LiveControl::Finish if idle => {+ self.queue.finish_if_empty();+ }+ LiveControl::Finish => self.queue.finish(),+ LiveControl::Continue => {}+ }+ Ok(())+ }++ pub fn run(+ &self,+ transcriber: &Transcriber,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ self.run_with(+ || transcriber.live_vad(),+ || {+ transcriber+ .load_model()+ .map_err(|_| VoiceError::ProcessingFailed)+ },+ |request| transcriber.transcribe_window(&request.samples),+ |pcm| {+ transcriber+ .transcribe(16_000, pcm)+ .map_err(|_| VoiceError::ProcessingFailed)+ },+ on_event,+ )+ }++ fn run_with(+ &self,+ make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,+ prepare: impl FnOnce() -> Result<()> + Send,+ mut decode: impl FnMut(&crate::voice::DecodeWindow) -> Result<Vec<AlignedWord>> + Send,+ decode_recorded: impl FnOnce(&[u8]) -> Result<String>,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ let _capture = self.start()?;+ let mut vad = make_vad()?;+ if vad.frame_size() != super::VAD_FRAME_SAMPLES {+ return Err(VoiceError::InvalidAudio);+ }+ let mut normalizer = StreamNormalizer::new(self.queue.format())?;+ let mut rolling = RollingRecognizer::default();+ let mut pcm = super::recorded::RecordedPcm::default();+ let mut accepted = 0_u64;+ let mut deferred = false;+ self.emit(LiveEvent::Ready, on_event)?;+ std::thread::scope(|scope| {+ let (requests, request_rx) = mpsc::sync_channel::<crate::voice::DecodeWindow>(1);+ let (results, result_rx) = mpsc::sync_channel(1);+ let decoder = scope.spawn(move || {+ let prepared = if self.queue.is_cancelled() {+ Err(VoiceError::Cancelled)+ } else {+ prepare()+ };+ let failed = prepared.is_err();+ if results.send(DecoderEvent::Prepared(prepared)).is_err() || failed {+ return;+ }+ while let Ok(request) = request_rx.recv() {+ let output = if self.queue.is_cancelled() {+ Err(VoiceError::Cancelled)+ } else {+ decode(&request)+ };+ if results.send(DecoderEvent::Window(request, output)).is_err() {+ break;+ }+ }+ });+ let mut finished = false;+ let mut prepared = false;+ let mut notified_stop = false;+ let processing = (|| -> Result<()> {+ loop {+ self.check()?;+ let chunk = self.queue.take();+ let had_chunk = chunk.is_some();+ let mut emit = |frame: &[f32], real: usize| -> Result<()> {+ let voiced = vad+ .is_speech(frame)+ .map_err(|_| VoiceError::ProcessingFailed)?;+ let start = accepted;+ accepted += real as u64;+ self.emit(+ LiveEvent::Audio {+ start,+ end: accepted,+ voiced,+ },+ on_event,+ )?;+ pcm.push(&frame[..real])?;+ if !deferred {+ match rolling.feed(&frame[..real], voiced) {+ Ok(()) => {}+ Err(VoiceError::BacklogFull) => {+ deferred = true;+ self.emit(LiveEvent::Deferred, on_event)?;+ }+ Err(error) => return Err(error),+ }+ }+ Ok(())+ };+ if let Some(chunk) = chunk {+ normalizer.push(&chunk, &mut emit)?;+ }+ if !finished && self.queue.is_drained() {+ normalizer.finish(&mut emit)?;+ if !deferred {+ rolling.finish();+ }+ finished = true;+ }+ if self.queue.is_closed() && !notified_stop {+ self.emit(LiveEvent::CaptureClosed, on_event)?;+ notified_stop = true;+ }+ match result_rx.try_recv() {+ Ok(DecoderEvent::Prepared(result)) => {+ result?;+ prepared = true;+ }+ Ok(DecoderEvent::Window(request, words)) => {+ if self.queue.is_cancelled() {+ return Err(VoiceError::Cancelled);+ }+ if !deferred {+ match words.and_then(|words| rolling.complete(&request, words)) {+ Ok(update) => {+ if let Some((start, end, text)) = update.commit {+ self.emit(+ LiveEvent::Commit { start, end, text },+ on_event,+ )?;+ }+ self.emit(+ LiveEvent::Hypothesis(update.hypothesis),+ on_event,+ )?;+ self.emit(+ LiveEvent::Coverage(update.covered_through),+ on_event,+ )?;+ }+ Err(+ VoiceError::UnstableTranscript+ | VoiceError::IncompleteAudio,+ ) => {+ deferred = true;+ self.emit(LiveEvent::Deferred, on_event)?;+ }+ Err(error) => return Err(error),+ }+ }+ }+ Err(TryRecvError::Empty) => {}+ Err(TryRecvError::Disconnected) => {+ return Err(VoiceError::ProcessingFailed);+ }+ }+ if prepared && finished && (deferred || rolling.is_settled()) {+ if !deferred {+ self.emit(LiveEvent::Coverage(accepted), on_event)?;+ }+ return Ok(());+ }+ if prepared+ && !deferred+ && let Some(request) = rolling.next_window()?+ {+ requests+ .try_send(request)+ .map_err(|_| VoiceError::ProcessingFailed)?;+ }+ if !had_chunk && !finished {+ self.emit(LiveEvent::Idle, on_event)?;+ }+ if !had_chunk {+ std::thread::sleep(Duration::from_millis(10));+ }+ }+ })();+ self.queue.finish();+ if !notified_stop {+ let _ = on_event(LiveEvent::CaptureClosed);+ }+ drop(requests);+ drop(result_rx);+ let exited = decoder.join().map_err(|_| VoiceError::ProcessingFailed);+ exited.and(processing)?;+ if deferred {+ drop(vad);+ drop(normalizer);+ self.check()?;+ let text = decode_recorded(pcm.bytes())?;+ self.check()?;+ if text.trim().is_empty() {+ return Err(VoiceError::IncompleteAudio);+ }+ self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;+ self.emit(LiveEvent::Coverage(accepted), on_event)?;+ }+ Ok(())+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use transcribe_rs::vad::EnergyVad;++ fn stream() -> LiveTranscription {+ LiveTranscription::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap()+ }++ fn vad() -> Result<Box<dyn Vad>> {+ Ok(Box::new(EnergyVad::new(+ super::super::VAD_FRAME_SAMPLES,+ 0.01,+ )))+ }++ #[test]+ fn capture_and_vad_preserve_opening_audio_while_model_loads() {+ let stream = stream();+ let (release, loading) = mpsc::sync_channel(1);+ let loaded = &AtomicBool::new(false);+ let mut coverage = 0;+ let mut recognized = false;+ let prepared = AtomicBool::new(false);+ let mut closed = 0;+ stream+ .run_with(+ || {+ prepared.store(true, Ordering::Release);+ vad()+ },+ move || {+ loading.recv_timeout(Duration::from_secs(3)).unwrap();+ loaded.store(true, Ordering::Release);+ Ok(())+ },+ |request| {+ assert!(loaded.load(Ordering::Acquire));+ assert_eq!(request.samples, vec![0.25; 10241]);+ Ok(vec![AlignedWord {+ text: "Opening words".into(),+ start: 0,+ end: 10241,+ }])+ },+ |_| panic!("This capture must stay live"),+ &mut |event| {+ match event {+ LiveEvent::Ready => {+ assert!(prepared.load(Ordering::Acquire));+ assert!(!loaded.load(Ordering::Acquire));+ for samples in [2560, 2560, 2560, 2561] {+ stream.feed(&[0, 32].repeat(samples))?;+ }+ }+ LiveEvent::Audio { end, .. } => {+ if end <= 10240 {+ assert!(!loaded.load(Ordering::Acquire));+ }+ if end == 10240 {+ stream.finish();+ release.send(()).unwrap();+ }+ }+ LiveEvent::CaptureClosed => closed += 1,+ LiveEvent::Commit { text, .. } => recognized |= text == "Opening words",+ LiveEvent::Coverage(end) => coverage = end,+ _ => {}+ }+ Ok(LiveControl::Continue)+ },+ )+ .unwrap();+ assert!(recognized);+ assert_eq!(coverage, 10241);+ assert_eq!(stream.submitted_samples(), 10241);+ assert_eq!(closed, 1);+ }++ #[test]+ fn slow_initialization_uses_existing_recorded_recovery_without_losing_audio() {+ let stream = stream();+ let (release, loading) = mpsc::sync_channel(1);+ let mut accepted = 0;+ let mut deferred = false;+ let mut replacement = None;+ let mut coverage = 0;+ stream+ .run_with(+ vad,+ move || {+ loading.recv_timeout(Duration::from_secs(3)).unwrap();+ Ok(())+ },+ |_| panic!("Initialization exceeded the existing live backlog limit"),+ |pcm| {+ assert_eq!(pcm, [0, 32].repeat(12 * 16000));+ Ok("All opening speech".into())+ },+ &mut |event| {+ match event {+ LiveEvent::Audio { end, .. } => accepted = end,+ LiveEvent::Idle => {+ if accepted < 12 * 16000 {+ for _ in 0..5 {+ stream.feed(&[0, 32].repeat(3200))?;+ }+ } else {+ assert!(deferred);+ stream.finish();+ release.send(()).unwrap();+ }+ }+ LiveEvent::Deferred => deferred = true,+ LiveEvent::ReplaceTranscript(text) => replacement = Some(text),+ LiveEvent::Coverage(end) => coverage = end,+ _ => {}+ }+ Ok(LiveControl::Continue)+ },+ )+ .unwrap();+ assert_eq!(replacement.as_deref(), Some("All opening speech"));+ assert_eq!(coverage, 12 * 16000);+ }++ #[test]+ fn stop_and_cancel_during_loading_close_capture_before_joining_the_worker() {+ for cancel in [false, true] {+ let stream = stream();+ let (release, loading) = mpsc::sync_channel(1);+ let exited = &AtomicBool::new(false);+ let started = &AtomicBool::new(false);+ let mut closed = 0;+ let result = stream.run_with(+ vad,+ move || {+ started.store(true, Ordering::Release);+ loading.recv_timeout(Duration::from_secs(3)).unwrap();+ exited.store(true, Ordering::Release);+ Ok(())+ },+ |_| panic!("No audio to decode"),+ |_| panic!("No recorded fallback"),+ &mut |event| {+ match event {+ LiveEvent::Ready if !cancel => stream.finish(),+ LiveEvent::Idle if cancel && started.load(Ordering::Acquire) => {+ stream.cancel()+ }+ LiveEvent::CaptureClosed => {+ assert!(!exited.load(Ordering::Acquire));+ closed += 1;+ release.send(()).unwrap();+ }+ LiveEvent::Ready | LiveEvent::Idle | LiveEvent::Coverage(0) => {}+ _ => panic!("Initialization emitted recognition"),+ }+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(+ result,+ if cancel {+ Err(VoiceError::Cancelled)+ } else {+ Ok(())+ }+ );+ assert_eq!(closed, 1);+ assert!(exited.load(Ordering::Acquire));+ }+ }++ #[test]+ fn failed_model_initialization_closes_capture_without_recognition() {+ let stream = stream();+ let mut closed = false;+ let result = stream.run_with(+ vad,+ || Err(VoiceError::ProcessingFailed),+ |_| panic!("Failed model must not decode"),+ |_| panic!("Failed model must not transcribe"),+ &mut |event| {+ match event {+ LiveEvent::Ready | LiveEvent::Idle => {}+ LiveEvent::CaptureClosed => closed = true,+ _ => panic!("Failed initialization emitted recognition"),+ }+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(result, Err(VoiceError::ProcessingFailed));+ assert!(closed);+ }++ #[test]+ fn failed_live_decode_continues_capture_and_replays_every_sample_once() {+ for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {+ let stream = stream();+ for _ in 0..3 {+ stream.feed(&[0, 32].repeat(3200)).unwrap();+ }+ let mut deferred = false;+ let mut closed = false;+ let mut final_text = Vec::new();+ let mut coverage = 0;+ stream+ .run_with(+ vad,+ || Ok(()),+ |_| Err(failure.clone()),+ |pcm| {+ assert_eq!(pcm, [0, 32].repeat(9600 + 1001));+ Ok("Complete recording".into())+ },+ &mut |event| {+ match event {+ LiveEvent::Deferred => {+ assert!(!deferred);+ assert!(!closed);+ deferred = true;+ stream.feed(&[0, 32].repeat(1001))?;+ stream.finish();+ }+ LiveEvent::CaptureClosed => {+ assert!(deferred);+ closed = true;+ }+ LiveEvent::ReplaceTranscript(text) => {+ assert!(closed);+ final_text.push(text);+ }+ LiveEvent::Coverage(end) => coverage = end,+ _ => {}+ }+ Ok(LiveControl::Continue)+ },+ )+ .unwrap();+ assert_eq!(final_text, ["Complete recording"]);+ assert_eq!(coverage, 10601);+ }+ }++ #[test]+ fn incomplete_or_cancelled_recovery_cannot_publish_text_or_coverage() {+ for cancelled in [false, true] {+ let stream = stream();+ stream.feed(&[0, 32].repeat(1001)).unwrap();+ stream.finish();+ let result = stream.run_with(+ vad,+ || Ok(()),+ |_| Err(VoiceError::UnstableTranscript),+ |_| {+ if cancelled {+ stream.cancel();+ Ok("late result".into())+ } else {+ Ok(String::new())+ }+ },+ &mut |event| {+ assert!(!matches!(+ event,+ LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)+ ));+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(+ result,+ Err(if cancelled {+ VoiceError::Cancelled+ } else {+ VoiceError::IncompleteAudio+ })+ );+ }+ }++ #[test]+ fn vad_keeps_running_during_decode_and_cancel_joins_native_work() {+ let stream = stream();+ for _ in 0..3 {+ stream.feed(&[1, 32].repeat(3200)).unwrap();+ }+ let decoded = &AtomicBool::new(false);+ let exited = &AtomicBool::new(false);+ let (release, decoding) = mpsc::sync_channel(1);+ let mut observed_during_decode = false;+ let mut fed = false;+ let result = stream.run_with(+ vad,+ || Ok(()),+ move |_| {+ decoded.store(true, Ordering::Release);+ decoding.recv_timeout(Duration::from_secs(3)).unwrap();+ exited.store(true, Ordering::Release);+ Ok(Vec::new())+ },+ |_| panic!("Cancelled decode must not transcribe"),+ &mut |event| {+ match event {+ LiveEvent::Idle if !fed && decoded.load(Ordering::Acquire) => {+ stream.feed(&[1, 32].repeat(3200))?;+ fed = true;+ }+ LiveEvent::Audio { end, .. } if end > 9600 => {+ assert!(!exited.load(Ordering::Acquire));+ observed_during_decode = true;+ return Ok(LiveControl::Cancel);+ }+ LiveEvent::CaptureClosed => release.send(()).unwrap(),+ _ => {}+ }+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(result, Err(VoiceError::Cancelled));+ assert!(observed_during_decode && exited.load(Ordering::Acquire));+ }+}diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rsindex 1d11862fc8..651de4da09 100644--- a/rust/crates/ensu/src/transcription/mod.rs+++ b/rust/crates/ensu/src/transcription/mod.rs@@ -1,8 +1,17 @@mod audio;-mod text;+mod graph_cache;+mod live;+mod normalize;+mod recorded;+mod session;mod transcriber;+mod vad;+pub use live::{LiveControl, LiveEvent, LiveTranscription};+pub(crate) use normalize::{StreamNormalizer, VAD_FRAME_SAMPLES};+pub use session::DictationState;pub use transcriber::Transcriber;+pub(crate) use vad::SileroV4Vad;#[derive(Debug, thiserror::Error)]pub enum TranscriptionError {@@ -36,6 +45,12 @@ impl From<transcribe_rs::TranscribeError> for TranscriptionError {}}+impl<R> From<ort::Error<R>> for TranscriptionError {+ fn from(value: ort::Error<R>) -> Self {+ Self::Transcribe(value.to_string())+ }+}+impl From<rubato::ResamplerConstructionError> for TranscriptionError {fn from(value: rubato::ResamplerConstructionError) -> Self {Self::InvalidAudio(value.to_string())diff --git a/rust/crates/ensu/src/transcription/normalize.rs b/rust/crates/ensu/src/transcription/normalize.rsnew file mode 100644index 0000000000..66e50b9d98--- /dev/null+++ b/rust/crates/ensu/src/transcription/normalize.rs@@ -0,0 +1,205 @@+use rubato::audioadapter_buffers::direct::InterleavedSlice;+use rubato::{Fft, FixedSync, Resampler, WindowFunction};++use crate::voice::{CaptureFormat, Result, VoiceError};++pub const VAD_FRAME_SAMPLES: usize = super::vad::SILERO_V4_FRAME_SAMPLES;++pub(crate) struct StreamNormalizer {+ format: CaptureFormat,+ resampler: Option<Fft<f32>>,+ input: Vec<f32>,+ frame: Vec<f32>,+ delay: usize,+ input_frames: u64,+ emitted: u64,+ finished: bool,+}++impl StreamNormalizer {+ pub fn new(format: CaptureFormat) -> Result<Self> {+ let format = format.validate()?;+ let resampler = if format.sample_rate == 16_000 {+ None+ } else {+ Some(+ Fft::<f32>::new_custom(+ format.sample_rate as usize,+ 16000,+ 1024,+ 1,+ 1,+ WindowFunction::BlackmanHarris2,+ FixedSync::Input,+ )+ .map_err(|_| VoiceError::InvalidAudio)?,+ )+ };+ let delay = resampler.as_ref().map_or(0, Resampler::output_delay);+ Ok(Self {+ format,+ resampler,+ input: Vec::with_capacity(2048),+ frame: Vec::with_capacity(VAD_FRAME_SAMPLES),+ delay,+ input_frames: 0,+ emitted: 0,+ finished: false,+ })+ }++ pub fn push(+ &mut self,+ pcm: &[u8],+ emit: &mut impl FnMut(&[f32], usize) -> Result<()>,+ ) -> Result<()> {+ let frame_bytes = self.format.bytes_per_frame();+ if self.finished || !pcm.len().is_multiple_of(frame_bytes) {+ return Err(VoiceError::InvalidAudio);+ }+ for frame in pcm.chunks_exact(frame_bytes) {+ let mono = frame+ .as_chunks::<2>()+ .0+ .iter()+ .map(|b| f32::from(i16::from_le_bytes(*b)) / 32768.0)+ .sum::<f32>()+ / f32::from(self.format.channels);+ self.input_frames += 1;+ if self.resampler.is_none() {+ self.emit_samples(&[mono], emit)?;+ } else {+ self.input.push(mono);+ if self+ .resampler+ .as_ref()+ .is_some_and(|r| self.input.len() == r.input_frames_next())+ {+ self.process_chunk(emit)?;+ }+ }+ }+ Ok(())+ }++ pub fn finish(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {+ if self.finished {+ return Ok(());+ }+ let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);+ while self.emitted < expected {+ let Some(resampler) = &self.resampler else {+ return Err(VoiceError::ProcessingFailed);+ };+ self.input.resize(resampler.input_frames_next(), 0.0);+ self.process_chunk(emit)?;+ }+ if !self.frame.is_empty() {+ let real = self.frame.len();+ self.frame.resize(VAD_FRAME_SAMPLES, 0.0);+ emit(&self.frame, real)?;+ self.frame.clear();+ }+ self.finished = true;+ Ok(())+ }++ fn process_chunk(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {+ let resampler = self.resampler.as_mut().ok_or(VoiceError::InvalidState)?;+ let input = InterleavedSlice::new(&self.input, 1, self.input.len())+ .map_err(|_| VoiceError::InvalidAudio)?;+ let output = resampler+ .process(&input, None)+ .map_err(|_| VoiceError::ProcessingFailed)?+ .take_data();+ self.input.clear();+ let skip = self.delay.min(output.len());+ self.delay -= skip;+ let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);+ let take = (expected - self.emitted).min((output.len() - skip) as u64) as usize;+ self.emit_samples(&output[skip..skip + take], emit)+ }++ fn emit_samples(+ &mut self,+ mut samples: &[f32],+ emit: &mut impl FnMut(&[f32], usize) -> Result<()>,+ ) -> Result<()> {+ self.emitted += samples.len() as u64;+ while !samples.is_empty() {+ let take = samples.len().min(VAD_FRAME_SAMPLES - self.frame.len());+ self.frame.extend_from_slice(&samples[..take]);+ samples = &samples[take..];+ if self.frame.len() == VAD_FRAME_SAMPLES {+ emit(&self.frame, VAD_FRAME_SAMPLES)?;+ self.frame.clear();+ }+ }+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn normalized(rate: u32, channels: u16, frames: usize, chunk: usize) -> Vec<f32> {+ let mut normalizer = StreamNormalizer::new(CaptureFormat {+ sample_rate: rate,+ channels,+ })+ .unwrap();+ let pcm: Vec<u8> = (0..frames)+ .flat_map(|i| {+ let value = ((std::f32::consts::TAU * 1000.0 * i as f32 / rate as f32).sin()+ * 10000.0) as i16;+ (0..channels).flat_map(move |_| value.to_le_bytes())+ })+ .collect();+ let mut output = Vec::new();+ let mut emit = |frame: &[f32], real: usize| {+ assert_eq!(frame.len(), VAD_FRAME_SAMPLES);+ output.extend_from_slice(&frame[..real]);+ Ok(())+ };+ for bytes in pcm.chunks(chunk * usize::from(channels) * 2) {+ normalizer.push(bytes, &mut emit).unwrap();+ }+ normalizer.finish(&mut emit).unwrap();+ normalizer.finish(&mut emit).unwrap();+ assert_eq!(output.len(), frames * 16000 / rate as usize);+ output+ }++ #[test]+ fn arbitrary_chunks_and_stereo_preserve_duration_and_final_tail() {+ for rate in [8000, 16000, 44100, 48000] {+ for frames in [1, 37, 1024, rate as usize + 19] {+ let expected = normalized(rate, 1, frames, 1024);+ for chunk in [1, 137, 4096] {+ assert_eq!(+ normalized(rate, 2, frames, chunk),+ expected,+ "{rate} Hz, {frames} frames, chunks {chunk}"+ );+ }+ }+ }+ }++ #[test]+ fn fft_delay_does_not_shift_source_timestamps() {+ let output = normalized(48000, 1, 48000, 257);+ let error = output[1600..14400]+ .iter()+ .enumerate()+ .map(|(i, value)| {+ let expected = (std::f32::consts::TAU * 1000.0 * (i + 1600) as f32 / 16000.0).sin()+ * 10000.0+ / 32768.0;+ (value - expected).abs()+ })+ .fold(0.0_f32, f32::max);+ assert!(error < 0.001, "{error}");+ }+}diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rsnew file mode 100644index 0000000000..cb8eb41377--- /dev/null+++ b/rust/crates/ensu/src/transcription/recorded.rs@@ -0,0 +1,214 @@+use std::time::Duration;++use transcribe_rs::vad::Vad;++use super::{LiveControl, LiveEvent, LiveTranscription, StreamNormalizer, Transcriber};+use crate::voice::{Result, VoiceError};++const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;++#[derive(Default)]+pub(super) struct RecordedPcm(Vec<u8>);++impl RecordedPcm {+ pub fn push(&mut self, samples: &[f32]) -> Result<()> {+ if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(self.0.len() / 2) {+ return Err(VoiceError::BacklogFull);+ }+ self.0.extend(samples.iter().flat_map(|sample| {+ ((sample * 32768.0).round().clamp(-32768.0, 32767.0) as i16).to_le_bytes()+ }));+ Ok(())+ }++ pub fn bytes(&self) -> &[u8] {+ &self.0+ }+}++impl Drop for RecordedPcm {+ fn drop(&mut self) {+ use zeroize::Zeroize;+ self.0.zeroize();+ }+}++impl LiveTranscription {+ pub fn run_recorded(+ &self,+ transcriber: &Transcriber,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ self.run_recorded_with(+ || transcriber.live_vad(),+ |pcm| {+ transcriber+ .transcribe(16_000, pcm)+ .map_err(|_| VoiceError::ProcessingFailed)+ },+ on_event,+ )+ }++ fn run_recorded_with(+ &self,+ make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,+ decode: impl FnOnce(&[u8]) -> Result<String>,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ let _capture = self.start()?;+ let mut vad = make_vad()?;+ let mut normalizer = StreamNormalizer::new(self.queue.format())?;+ self.emit(LiveEvent::Ready, on_event)?;+ let mut pcm = RecordedPcm::default();+ let mut accepted = 0_u64;+ let mut heard_speech = false;+ let mut notified_stop = false;+ loop {+ self.check()?;+ let chunk = self.queue.take();+ let had_chunk = chunk.is_some();+ let mut emit = |frame: &[f32], real: usize| -> Result<()> {+ let voiced = vad+ .is_speech(frame)+ .map_err(|_| VoiceError::ProcessingFailed)?;+ heard_speech |= voiced;+ let start = accepted;+ accepted += real as u64;+ self.emit(+ LiveEvent::Audio {+ start,+ end: accepted,+ voiced,+ },+ on_event,+ )?;+ pcm.push(&frame[..real])+ };+ if let Some(chunk) = chunk {+ normalizer.push(&chunk, &mut emit)?;+ }+ let drained = self.queue.is_drained();+ if drained {+ normalizer.finish(&mut emit)?;+ }+ if self.queue.is_closed() && !notified_stop {+ self.emit(LiveEvent::CaptureClosed, on_event)?;+ notified_stop = true;+ }+ if drained {+ break;+ }+ if !had_chunk {+ self.emit(LiveEvent::Idle, on_event)?;+ std::thread::sleep(Duration::from_millis(10));+ }+ }+ drop(vad);+ drop(normalizer);+ self.check()?;+ if heard_speech {+ let text = decode(pcm.bytes())?;+ self.check()?;+ if !text.trim().is_empty() {+ self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;+ }+ }+ self.emit(LiveEvent::Coverage(accepted), on_event)?;+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::voice::CaptureFormat;+ use transcribe_rs::vad::EnergyVad;++ #[test]+ fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {+ let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);+ pcm.push(&[0.25]).unwrap();+ assert_eq!(pcm.bytes().len(), MAX_RECORDED_SAMPLES * 2);+ assert_eq!(pcm.push(&[0.5]), Err(VoiceError::BacklogFull));+ assert_eq!(+ &pcm.bytes()[pcm.bytes().len() - 2..],+ &8192_i16.to_le_bytes()+ );+ }++ #[test]+ fn recorded_text_arrives_once_after_close_with_partial_tail() {+ let stream = LiveTranscription::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap();+ stream.feed(&[0, 32].repeat(1001)).unwrap();+ stream.finish();+ let mut closed = false;+ let mut commits = Vec::new();+ let mut covered = 0;+ stream+ .run_recorded_with(+ || {+ Ok(Box::new(EnergyVad::new(+ super::super::VAD_FRAME_SAMPLES,+ 0.01,+ )))+ },+ |pcm| {+ assert_eq!(pcm.len(), 1001 * 2);+ assert_eq!(pcm, [0, 32].repeat(1001));+ Ok("dictated words".into())+ },+ &mut |event| {+ match event {+ LiveEvent::CaptureClosed => closed = true,+ LiveEvent::ReplaceTranscript(text) => {+ assert!(closed);+ commits.push(text);+ }+ LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),+ LiveEvent::Coverage(end) => covered = end,+ _ => {}+ }+ Ok(LiveControl::Continue)+ },+ )+ .unwrap();+ assert_eq!(commits, ["dictated words"]);+ assert_eq!(covered, 1001);+ }++ #[test]+ fn cancellation_during_batch_decode_does_not_publish_text() {+ let stream = LiveTranscription::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap();+ stream.feed(&[0, 32].repeat(512)).unwrap();+ stream.finish();+ let result = stream.run_recorded_with(+ || {+ Ok(Box::new(EnergyVad::new(+ super::super::VAD_FRAME_SAMPLES,+ 0.01,+ )))+ },+ |_| {+ stream.cancel();+ Ok("late result".into())+ },+ &mut |event| {+ assert!(!matches!(+ event,+ LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)+ ));+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(result, Err(VoiceError::Cancelled));+ }+}diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rsnew file mode 100644index 0000000000..4e1a0e5df0--- /dev/null+++ b/rust/crates/ensu/src/transcription/session.rs@@ -0,0 +1,166 @@+use super::{LiveControl, LiveEvent};+use crate::voice;++#[derive(Default)]+pub struct DictationState {+ pub session: voice::VoiceSession,+ text: String,+ hypothesis: String,+ last_commit_end: u64,+}+impl DictationState {+ pub fn new(live: bool) -> Self {+ Self {+ session: if live {+ voice::VoiceSession::default()+ } else {+ voice::VoiceSession::recorded()+ },+ ..Self::default()+ }+ }++ pub fn text(&self) -> &str {+ &self.text+ }++ pub fn hypothesis(&self) -> String {+ format!(+ "{}{}",+ voice::boundary_separator(&self.text, &self.hypothesis),+ self.hypothesis+ )+ }++ pub fn apply(+ &mut self,+ event: LiveEvent,+ finish_if_empty: impl FnOnce() -> bool,+ ) -> voice::Result<LiveControl> {+ if !self.session.worker_active() || self.session.phase() == voice::VoicePhase::Closed {+ return Ok(LiveControl::Cancel);+ }+ match event {+ LiveEvent::Ready => {}+ LiveEvent::Audio { start, end, voiced } => {+ if self.session.stop_reason().is_some() {+ self.session.observe_drained_audio(start, end)?;+ } else if let Some(reason) =+ self.session+ .observe_audio(start, end, voiced.then_some(end))?+ {+ self.session.stop(reason)?;+ return Ok(LiveControl::Finish);+ }+ }+ LiveEvent::Hypothesis(text) => {+ self.session.observe_recognition(!text.trim().is_empty())?;+ self.hypothesis = text;+ }+ LiveEvent::Commit { start, end, text } => {+ if end <= start || start < self.last_commit_end {+ return Err(voice::VoiceError::InvalidAudio);+ }+ self.last_commit_end = end;+ self.session.observe_recognition(!text.trim().is_empty())?;+ self.text+ .push_str(voice::boundary_separator(&self.text, &text));+ self.text.push_str(&text);+ self.hypothesis.clear();+ }+ LiveEvent::ReplaceTranscript(text) => {+ if text.trim().is_empty() {+ return Err(voice::VoiceError::IncompleteAudio);+ }+ self.session.observe_recognition(true)?;+ self.text = text;+ self.hypothesis.clear();+ self.last_commit_end = u64::MAX;+ }+ LiveEvent::Deferred => self.session.defer_recognition(),+ LiveEvent::Coverage(end) => self.session.recognize_through(end)?,+ LiveEvent::Idle => {+ if self.session.silence_elapsed() && finish_if_empty() {+ self.session.stop(voice::StopReason::Silence)?;+ }+ }+ LiveEvent::CaptureClosed => {+ if self.session.stop_reason().is_none() {+ self.session.stop(voice::StopReason::Silence)?;+ }+ }+ }+ Ok(LiveControl::Continue)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn event(state: &mut DictationState, event: LiveEvent) {+ state.apply(event, || true).unwrap();+ }++ #[test]+ fn queued_audio_is_processed_before_silence_can_stop_capture() {+ let mut state = DictationState::new(false);+ state.session.begin_capture().unwrap();+ event(+ &mut state,+ LiveEvent::Audio {+ start: 0,+ end: 16000,+ voiced: true,+ },+ );+ event(+ &mut state,+ LiveEvent::Audio {+ start: 16000,+ end: 64000,+ voiced: false,+ },+ );+ state.apply(LiveEvent::Idle, || false).unwrap();+ assert!(state.session.stop_reason().is_none());+ event(+ &mut state,+ LiveEvent::Audio {+ start: 64000,+ end: 65000,+ voiced: true,+ },+ );+ event(&mut state, LiveEvent::Idle);+ assert!(state.session.stop_reason().is_none());+ event(+ &mut state,+ LiveEvent::Audio {+ start: 65000,+ end: 113000,+ voiced: false,+ },+ );+ event(&mut state, LiveEvent::Idle);+ assert_eq!(+ state.session.stop_reason(),+ Some(voice::StopReason::Silence)+ );+ }++ #[test]+ fn preview_preserves_native_spaces_and_unspaced_scripts() {+ for (stable, partial, expected) in [+ ("明", "天", "明天"),+ ("hello", "world", "hello world"),+ ("hello ", "world", "hello world"),+ ] {+ let mut state = DictationState::new(true);+ state.session.begin_capture().unwrap();+ event(&mut state, LiveEvent::ReplaceTranscript(stable.into()));+ event(&mut state, LiveEvent::Hypothesis(partial.into()));+ assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);+ }+ }+}diff --git a/rust/crates/ensu/src/transcription/text.rs b/rust/crates/ensu/src/transcription/text.rsdeleted file mode 100644index 781fbbd952..0000000000--- a/rust/crates/ensu/src/transcription/text.rs+++ /dev/null@@ -1,81 +0,0 @@-use regex::{Regex, regex};-use std::sync::LazyLock;--const FILLER_WORDS: &[&str] = &[- "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha",- "ehh",-];--#[expect(- clippy::expect_used,- reason = "Filler words are escaped before insertion into the fixed regex"-)]-static FILLER_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {- FILLER_WORDS- .iter()- .map(|word| {- Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word)))- .expect("escaped filler word forms a valid regex")- })- .collect()-});--pub fn filter_transcription_output(text: &str) -> String {- let mut filtered = text.to_string();-- for pattern in FILLER_PATTERNS.iter() {- filtered = pattern.replace_all(&filtered, "").to_string();- }-- filtered = collapse_stutters(&filtered);- filtered = regex!(r"\s{2,}").replace_all(&filtered, " ").to_string();- filtered.trim().to_string()-}--fn collapse_stutters(text: &str) -> String {- let words = text.split_whitespace().collect::<Vec<_>>();- if words.is_empty() {- return text.to_string();- }-- let mut result = Vec::new();- let mut i = 0;-- while i < words.len() {- let word = words[i];- let word_lower = word.to_lowercase();-- if word_lower.len() <= 2 && word_lower.chars().all(char::is_alphabetic) {- let mut count = 1;- while i + count < words.len() && words[i + count].to_lowercase() == word_lower {- count += 1;- }-- result.push(word);- i += if count >= 3 { count } else { 1 };- } else {- result.push(word);- i += 1;- }- }-- result.join(" ")-}--#[cfg(test)]-mod tests {- use super::filter_transcription_output;-- #[test]- fn removes_filler_words() {- assert_eq!(- filter_transcription_output("um hello uh world"),- "hello world"- );- }-- #[test]- fn collapses_repeated_short_stutters() {- assert_eq!(filter_transcription_output("I I I think"), "I think");- }-}diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rsindex cbfd111cef..16a34ba8ee 100644--- a/rust/crates/ensu/src/transcription/transcriber.rs+++ b/rust/crates/ensu/src/transcription/transcriber.rs@@ -3,31 +3,105 @@ use std::sync::{Mutex, MutexGuard, PoisonError};use transcribe_rs::onnx::Quantization;use transcribe_rs::onnx::parakeet::{ParakeetModel, ParakeetParams, TimestampGranularity};+use transcribe_rs::{OrtAccelerator, get_ort_accelerator};+use super::graph_cache::{GRAPHS, GraphCache, LoadPlan, SOURCES};use crate::transcription::audio::extract_speech_from_pcm16;-use crate::transcription::text::filter_transcription_output;use crate::transcription::{Result, TranscriptionError};pub struct Transcriber {model_dir: PathBuf,vad_model_path: PathBuf,loaded: Mutex<Option<ParakeetModel>>,+ graph_cache: Option<GraphCache>,}impl Transcriber {+ pub(crate) fn live_vad(&self) -> crate::voice::Result<Box<dyn transcribe_rs::vad::Vad>> {+ super::SileroV4Vad::new(&self.vad_model_path)+ .map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>)+ .map_err(|_| crate::voice::VoiceError::ProcessingFailed)+ }++ pub(crate) fn transcribe_window(+ &self,+ samples: &[f32],+ ) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {+ use crate::voice::{AlignedWord, MAX_WINDOW_SAMPLES, VoiceError};+ if samples.is_empty()+ || samples.len() > MAX_WINDOW_SAMPLES+ || samples.iter().any(|s| !s.is_finite())+ {+ return Err(VoiceError::InvalidAudio);+ }+ let mut loaded = self.lock();+ let model = ensure_loaded(+ &mut loaded,+ &self.model_dir,+ self.graph_cache.as_ref(),+ true,+ )+ .map_err(|_| VoiceError::ProcessingFailed)?;+ let result = model+ .transcribe_with(+ samples,+ &ParakeetParams {+ timestamp_granularity: Some(TimestampGranularity::Word),+ ..Default::default()+ },+ )+ .map_err(|_| VoiceError::ProcessingFailed)?;+ result+ .segments+ .ok_or(VoiceError::UnstableTranscript)?+ .into_iter()+ .map(|word| {+ if !word.start.is_finite()+ || !word.end.is_finite()+ || word.end < word.start+ || word.start * 16000.0 > samples.len() as f32+ {+ return Err(VoiceError::UnstableTranscript);+ }+ Ok(AlignedWord {+ text: word.text,+ start: (word.start.max(0.0) * 16000.0).round() as u64,+ end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples.len() as u64),+ })+ })+ .collect()+ }+pub fn new(model_dir: impl Into<PathBuf>, vad_model_path: impl Into<PathBuf>) -> Self {Self {model_dir: model_dir.into(),vad_model_path: vad_model_path.into(),loaded: Mutex::new(None),+ graph_cache: None,}}+ pub fn with_cache(+ model_dir: impl Into<PathBuf>,+ vad_model_path: impl Into<PathBuf>,+ cache_directory: impl Into<PathBuf>,+ device_identity: String,+ ) -> Self {+ let mut transcriber = Self::new(model_dir, vad_model_path);+ transcriber.graph_cache = Some(GraphCache::new(cache_directory.into(), device_identity));+ transcriber+ }+pub fn load_model(&self) -> Result<()> {if !self.model_dir.is_dir() {return Err(TranscriptionError::NotDownloaded);}- ensure_loaded(&mut self.lock(), &self.model_dir)?;+ ensure_loaded(+ &mut self.lock(),+ &self.model_dir,+ self.graph_cache.as_ref(),+ true,+ )?;Ok(())}@@ -52,7 +126,12 @@ impl Transcriber {}let mut loaded = self.lock();- let model = ensure_loaded(&mut loaded, &self.model_dir)?;+ let model = ensure_loaded(+ &mut loaded,+ &self.model_dir,+ self.graph_cache.as_ref(),+ false,+ )?;let result = model.transcribe_with(&speech,&ParakeetParams {@@ -61,7 +140,7 @@ impl Transcriber {},)?;- Ok(filter_transcription_output(&result.text))+ Ok(result.text.trim().to_owned())}fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {@@ -72,9 +151,82 @@ impl Transcriber {fn ensure_loaded<'a>(loaded: &'a mut Option<ParakeetModel>,model_dir: &Path,+ cache: Option<&GraphCache>,+ may_build: bool,) -> Result<&'a mut ParakeetModel> {match loaded {Some(model) => Ok(model),- slot => Ok(slot.insert(ParakeetModel::load(model_dir, &Quantization::Int8)?)),+ slot => {+ for name in SOURCES {+ if !model_dir.join(name).is_file() {+ return Err(TranscriptionError::NotDownloaded);+ }+ }+ let cache = cache.filter(|_| {+ get_ort_accelerator() == OrtAccelerator::CpuOnly+ || OrtAccelerator::available() == [OrtAccelerator::CpuOnly]+ });+ let result = if let Some(cache) = cache {+ let configuration = format!(+ "{};cpu;parallel=true;intra=auto;threads={}",+ ort::info(),+ std::thread::available_parallelism()+ .map(std::num::NonZeroUsize::get)+ .unwrap_or(1)+ );+ cache.load(model_dir, &configuration, may_build, |plan| {+ load_plan(model_dir, plan)+ })+ } else {+ load_plan(model_dir, LoadPlan::Source { save_to: None })+ };+ Ok(slot.insert(result?))+ }+ }+}++fn load_plan(model_dir: &Path, plan: LoadPlan<'_>) -> Result<ParakeetModel> {+ use ort::session::builder::GraphOptimizationLevel;+ let directory = match plan {+ LoadPlan::Source { save_to: None } => model_dir,+ LoadPlan::Cached { directory } => directory,+ LoadPlan::Source {+ save_to: Some(directory),+ } => {+ for name in GRAPHS {+ let session = ort::session::Session::builder()?+ .with_optimization_level(GraphOptimizationLevel::Level3)?+ .with_optimized_model_path(directory.join(name))?+ .with_parallel_execution(true)?+ .with_execution_providers([ort::ep::CPU::default().build()])?+ .commit_from_file(model_dir.join(name))?;+ drop(session);+ }+ directory+ }+ };+ Ok(ParakeetModel::load(directory, &Quantization::Int8)?)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn missing_quantized_graph_never_falls_back_to_full_precision() {+ let directory = tempfile::tempdir().unwrap();+ for name in [+ "encoder-model.onnx",+ "decoder_joint-model.onnx",+ "nemo128.onnx",+ "vocab.txt",+ ] {+ std::fs::write(directory.path().join(name), b"not a model").unwrap();+ }+ let transcriber = Transcriber::new(directory.path(), directory.path().join("vad.onnx"));+ assert!(matches!(+ transcriber.load_model(),+ Err(TranscriptionError::NotDownloaded)+ ));}}diff --git a/rust/crates/ensu/src/transcription/vad.rs b/rust/crates/ensu/src/transcription/vad.rsnew file mode 100644index 0000000000..8869d0139c--- /dev/null+++ b/rust/crates/ensu/src/transcription/vad.rs@@ -0,0 +1,106 @@+use ort::{+ inputs,+ session::{Session, builder::GraphOptimizationLevel},+ value::TensorRef,+};+use std::path::Path;+use transcribe_rs::{TranscribeError, vad::Vad};++pub(super) const SILERO_V4_FRAME_SAMPLES: usize = 512;++pub(crate) struct SileroV4Vad {+ session: Session,+ hidden: [f32; 128],+ cell: [f32; 128],+}++fn inference(error: impl std::fmt::Display) -> TranscribeError {+ TranscribeError::Inference(format!("Silero v4: {error}"))+}++impl SileroV4Vad {+ pub fn new(path: impl AsRef<Path>) -> Result<Self, TranscribeError> {+ let session = Session::builder()+ .map_err(inference)?+ .with_optimization_level(GraphOptimizationLevel::Level3)+ .map_err(inference)?+ .with_intra_threads(1)+ .map_err(inference)?+ .with_inter_threads(1)+ .map_err(inference)?+ .commit_from_file(path)+ .map_err(inference)?;+ Ok(Self {+ session,+ hidden: [0.0; 128],+ cell: [0.0; 128],+ })+ }++ fn speech_probability(&mut self, frame: &[f32]) -> Result<f32, TranscribeError> {+ validate_frame(frame)?;+ let sample_rate = [16000_i64];+ let outputs = self.session.run(inputs![+ "input" => TensorRef::from_array_view(([1, SILERO_V4_FRAME_SAMPLES], frame)).map_err(inference)?,+ "sr" => TensorRef::from_array_view(([1], sample_rate.as_slice())).map_err(inference)?,+ "h" => TensorRef::from_array_view(([2,1,64], self.hidden.as_slice())).map_err(inference)?,+ "c" => TensorRef::from_array_view(([2,1,64], self.cell.as_slice())).map_err(inference)?,+ ]).map_err(inference)?;+ let (shape, probability) = outputs+ .get("output")+ .ok_or_else(|| inference("missing probability"))?+ .try_extract_tensor::<f32>()+ .map_err(inference)?;+ if shape.as_ref() != [1, 1]+ || probability.len() != 1+ || !probability[0].is_finite()+ || !(0.0..=1.0).contains(&probability[0])+ {+ return Err(inference("invalid probability"));+ }+ let probability = probability[0];+ let (h_shape, hidden) = outputs+ .get("hn")+ .ok_or_else(|| inference("missing hidden state"))?+ .try_extract_tensor::<f32>()+ .map_err(inference)?;+ let (c_shape, cell) = outputs+ .get("cn")+ .ok_or_else(|| inference("missing cell state"))?+ .try_extract_tensor::<f32>()+ .map_err(inference)?;+ if h_shape.as_ref() != [2, 1, 64]+ || c_shape.as_ref() != [2, 1, 64]+ || hidden.len() != 128+ || cell.len() != 128+ || hidden.iter().chain(cell).any(|x| !x.is_finite())+ {+ return Err(inference("invalid recurrent state"));+ }+ self.hidden.copy_from_slice(hidden);+ self.cell.copy_from_slice(cell);+ Ok(probability)+ }+}++fn validate_frame(frame: &[f32]) -> Result<(), TranscribeError> {+ if frame.len() != SILERO_V4_FRAME_SAMPLES || frame.iter().any(|s| !s.is_finite()) {+ return Err(TranscribeError::Audio(+ "Silero v4 requires 512 finite samples at 16 kHz".into(),+ ));+ }+ Ok(())+}++impl Vad for SileroV4Vad {+ fn frame_size(&self) -> usize {+ SILERO_V4_FRAME_SAMPLES+ }+ fn is_speech(&mut self, frame: &[f32]) -> Result<bool, TranscribeError> {+ Ok(self.speech_probability(frame)? > 0.5)+ }+ fn reset(&mut self) {+ self.hidden.fill(0.0);+ self.cell.fill(0.0);+ }+}diff --git a/rust/crates/ensu/src/voice/audio.rs b/rust/crates/ensu/src/voice/audio.rsnew file mode 100644index 0000000000..5ee6a89eb2--- /dev/null+++ b/rust/crates/ensu/src/voice/audio.rs@@ -0,0 +1,121 @@+use std::collections::VecDeque;++use super::{Result, VoiceError};++pub const VOICE_SAMPLE_RATE: u64 = 16_000;+pub const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;+pub const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;+pub const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;++#[derive(Debug)]+pub struct AudioRing {+ samples: VecDeque<f32>,+ start: u64,+ processed: u64,+}++impl Default for AudioRing {+ fn default() -> Self {+ Self {+ samples: VecDeque::with_capacity(MAX_RING_SAMPLES),+ start: 0,+ processed: 0,+ }+ }+}++impl AudioRing {+ pub fn push(&mut self, offset: u64, samples: &[f32]) -> Result<()> {+ if offset != self.accepted() || samples.iter().any(|v| !v.is_finite()) {+ return Err(VoiceError::InvalidAudio);+ }+ let end = offset+ .checked_add(samples.len() as u64)+ .ok_or(VoiceError::InvalidAudio)?;+ if end - self.processed > MAX_BACKLOG_SAMPLES+ || self.samples.len().saturating_add(samples.len()) > MAX_RING_SAMPLES+ {+ return Err(VoiceError::BacklogFull);+ }+ self.samples.extend(samples);+ Ok(())+ }++ pub fn window(&self, start: u64, end: u64) -> Result<Vec<f32>> {+ if start < self.start+ || end > self.accepted()+ || end < start+ || end - start > MAX_WINDOW_SAMPLES as u64+ {+ return Err(VoiceError::InvalidAudio);+ }+ Ok(self+ .samples+ .iter()+ .skip((start - self.start) as usize)+ .take((end - start) as usize)+ .copied()+ .collect())+ }++ pub fn acknowledge(&mut self, processed: u64, retain_from: u64) -> Result<()> {+ if processed < self.processed+ || processed > self.accepted()+ || retain_from < self.start+ || retain_from > processed+ {+ return Err(VoiceError::InvalidAudio);+ }+ self.samples.drain(..(retain_from - self.start) as usize);+ self.start = retain_from;+ self.processed = processed;+ Ok(())+ }++ pub fn accepted(&self) -> u64 {+ self.start + self.samples.len() as u64+ }+ pub fn retained_from(&self) -> u64 {+ self.start+ }+ pub fn processed(&self) -> u64 {+ self.processed+ }+ #[cfg(test)]+ pub(super) fn retained_samples(&self) -> usize {+ self.samples.len()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn saturation_never_overwrites_unrecognized_audio() {+ let mut ring = AudioRing::default();+ ring.push(0, &vec![0.25; MAX_BACKLOG_SAMPLES as usize])+ .unwrap();+ assert_eq!(+ ring.push(ring.accepted(), &[0.5]),+ Err(VoiceError::BacklogFull)+ );+ assert_eq!(ring.accepted(), MAX_BACKLOG_SAMPLES);+ assert_eq!(ring.window(0, 1).unwrap(), vec![0.25]);+ ring.acknowledge(ring.accepted(), ring.accepted() - 8000)+ .unwrap();+ ring.push(ring.accepted(), &[0.5]).unwrap();+ assert_eq!(ring.retained_samples(), 8001);+ assert_eq!(ring.accepted(), MAX_BACKLOG_SAMPLES + 1);+ }++ #[test]+ fn invalid_offsets_and_nan_are_rejected_atomically() {+ let mut ring = AudioRing::default();+ assert_eq!(ring.push(1, &[0.0]), Err(VoiceError::InvalidAudio));+ assert_eq!(ring.push(0, &[f32::NAN]), Err(VoiceError::InvalidAudio));+ assert_eq!(ring.accepted(), 0);+ ring.push(0, &[0.1, 0.2]).unwrap();+ assert_eq!(ring.acknowledge(3, 0), Err(VoiceError::InvalidAudio));+ }+}diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rsnew file mode 100644index 0000000000..544e8941a3--- /dev/null+++ b/rust/crates/ensu/src/voice/capture.rs@@ -0,0 +1,185 @@+use std::collections::VecDeque;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Mutex, MutexGuard, PoisonError};++use super::{Result, VoiceError};++#[derive(Clone, Copy, Debug)]+pub struct CaptureFormat {+ pub sample_rate: u32,+ pub channels: u16,+}++impl CaptureFormat {+ pub fn validate(self) -> Result<Self> {+ if !(8_000..=48_000).contains(&self.sample_rate) || !matches!(self.channels, 1 | 2) {+ return Err(VoiceError::InvalidAudio);+ }+ Ok(self)+ }++ pub fn bytes_per_frame(self) -> usize {+ usize::from(self.channels) * 2+ }+}++#[derive(Default)]+struct Pending {+ chunks: VecDeque<Vec<u8>>,+ bytes: usize,+ closed: bool,+}++pub struct CaptureQueue {+ format: CaptureFormat,+ pending: Mutex<Pending>,+ failed: AtomicBool,+ cancelled: AtomicBool,+ submitted_frames: AtomicU64,+}++impl CaptureQueue {+ pub fn new(format: CaptureFormat) -> Result<Self> {+ Ok(Self {+ format: format.validate()?,+ pending: Mutex::new(Pending::default()),+ failed: AtomicBool::new(false),+ cancelled: AtomicBool::new(false),+ submitted_frames: AtomicU64::new(0),+ })+ }++ pub fn format(&self) -> CaptureFormat {+ self.format+ }++ pub fn push(&self, pcm: &[u8]) -> Result<()> {+ if self.cancelled.load(Ordering::Acquire) {+ return Err(VoiceError::Cancelled);+ }+ if self.failed.load(Ordering::Acquire) {+ return Err(VoiceError::BacklogFull);+ }+ let frame_bytes = self.format.bytes_per_frame();+ if pcm.is_empty()+ || !pcm.len().is_multiple_of(frame_bytes)+ || pcm.len() / frame_bytes > self.format.sample_rate as usize / 5+ {+ self.failed.store(true, Ordering::Release);+ return Err(VoiceError::InvalidAudio);+ }+ let owned = pcm.to_vec();+ let mut pending = self.lock();+ if pending.closed {+ return Err(VoiceError::InvalidState);+ }+ self.submitted_frames+ .fetch_add((pcm.len() / frame_bytes) as u64, Ordering::AcqRel);+ if pending.bytes + pcm.len() > self.format.sample_rate as usize * frame_bytes * 2+ || pending.chunks.len() >= 128+ {+ self.failed.store(true, Ordering::Release);+ return Err(VoiceError::BacklogFull);+ }+ pending.bytes += pcm.len();+ pending.chunks.push_back(owned);+ Ok(())+ }++ pub fn take(&self) -> Option<Vec<u8>> {+ let mut pending = self.lock();+ let chunk = pending.chunks.pop_front()?;+ pending.bytes -= chunk.len();+ Some(chunk)+ }++ pub fn finish(&self) {+ self.lock().closed = true;+ }++ pub fn finish_if_empty(&self) -> bool {+ let mut pending = self.lock();+ if !pending.chunks.is_empty() {+ return false;+ }+ pending.closed = true;+ true+ }++ pub fn cancel(&self) {+ self.cancelled.store(true, Ordering::Release);+ self.finish();+ }++ pub fn is_cancelled(&self) -> bool {+ self.cancelled.load(Ordering::Acquire)+ }++ pub fn failed(&self) -> bool {+ self.failed.load(Ordering::Acquire)+ }++ pub fn is_drained(&self) -> bool {+ let pending = self.lock();+ pending.closed && pending.chunks.is_empty()+ }++ pub fn is_closed(&self) -> bool {+ self.lock().closed+ }++ pub fn submitted_frames(&self) -> u64 {+ self.submitted_frames.load(Ordering::Acquire)+ }++ pub fn clear(&self) {+ let mut pending = self.lock();+ pending.closed = true;+ pending.chunks.clear();+ pending.bytes = 0;+ }++ fn lock(&self) -> MutexGuard<'_, Pending> {+ self.pending.lock().unwrap_or_else(PoisonError::into_inner)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn queue_bounds_bytes_and_saturation_is_sticky() {+ let queue = CaptureQueue::new(CaptureFormat {+ sample_rate: 48000,+ channels: 2,+ })+ .unwrap();+ let chunk = vec![7; 48000 / 5 * 4];+ for _ in 0..10 {+ queue.push(&chunk).unwrap();+ }+ assert_eq!(queue.push(&chunk), Err(VoiceError::BacklogFull));+ assert_eq!(queue.lock().bytes, 96000 * 4);+ assert_eq!(queue.take().unwrap(), chunk);+ assert!(queue.failed());+ assert_eq!(queue.push(&chunk), Err(VoiceError::BacklogFull));+ }++ #[test]+ fn countdown_cannot_skip_queued_speech_and_stop_drains_tail() {+ let queue = CaptureQueue::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap();+ queue.push(&[1, 0]).unwrap();+ assert!(!queue.finish_if_empty());+ assert_eq!(queue.take(), Some(vec![1, 0]));+ assert!(queue.finish_if_empty());+ assert!(queue.is_drained());+ assert_eq!(queue.push(&[1, 0]), Err(VoiceError::InvalidState));+ queue.cancel();+ assert_eq!(queue.push(&[1, 0]), Err(VoiceError::Cancelled));+ }+}diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rsnew file mode 100644index 0000000000..983d956d39--- /dev/null+++ b/rust/crates/ensu/src/voice/composer.rs@@ -0,0 +1,265 @@+use std::ops::Range;++use super::{Result, VoiceError};++pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {+ if start > end {+ return Err(VoiceError::InvalidTextRange);+ }+ let mut units = 0_u32;+ let mut from = None;+ for (byte, character) in text+ .char_indices()+ .chain(std::iter::once((text.len(), '\0')))+ {+ if units == start {+ from = Some(byte);+ }+ if units == end {+ return Ok(from.ok_or(VoiceError::InvalidTextRange)?..byte);+ }+ units = units+ .checked_add(character.len_utf16() as u32)+ .ok_or(VoiceError::InvalidTextRange)?;+ }+ Err(VoiceError::InvalidTextRange)+}++pub fn boundary_separator(left: &str, right: &str) -> &'static str {+ fn unspaced(c: char) -> bool {+ matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)+ }+ let left_edge = left+ .chars()+ .rev()+ .find(|c| {+ !regex::regex!(r"\A[\p{M}\p{Join_Control}]\z").is_match(c.encode_utf8(&mut [0; 4]))+ })+ .or_else(|| left.chars().next());+ let right_edge = right.chars().next();+ match (left_edge, right_edge) {+ (Some(a), Some(b))+ if (a.is_alphanumeric()+ || matches!(+ a,+ '.' | '!' | '?' | ':' | ';' | ',' | '।' | '॥' | '؟' | '؛' | '،'+ ))+ && b.is_alphanumeric()+ && !unspaced(a)+ && !unspaced(b) =>+ {+ " "+ }+ _ => "",+ }+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ComposerAnchor {+ pub destination: String,+ pub revision: u64,+ pub start_utf16: u32,+ pub end_utf16: u32,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct VoiceInsertion {+ pub anchor: ComposerAnchor,+ pub text: String,+}++pub fn preview_insertion(+ text: &str,+ anchor: &ComposerAnchor,+ voice_text: &str,+) -> Result<(String, u32)> {+ let range = utf16_range(text, anchor.start_utf16, anchor.end_utf16)?;+ if voice_text.is_empty() {+ return Ok((text.to_owned(), anchor.end_utf16));+ }+ let before = &text[..range.start];+ let after = &text[range.end..];+ let prefix = boundary_separator(before, voice_text);+ let suffix = boundary_separator(voice_text, after);+ let caret = before.encode_utf16().count()+ + prefix.encode_utf16().count()+ + voice_text.encode_utf16().count()+ + suffix.encode_utf16().count();+ Ok((+ format!("{before}{prefix}{voice_text}{suffix}{after}"),+ caret as u32,+ ))+}++#[derive(Debug, Default)]+pub struct ComposerTransfer {+ receipt: Option<(VoiceInsertion, u32)>,+}++impl ComposerTransfer {+ pub fn apply(+ &mut self,+ insertion: &VoiceInsertion,+ destination: &str,+ revision: &mut u64,+ text: &mut String,+ foreground: bool,+ composing: bool,+ ) -> Result<u32> {+ if let Some((applied, receipt)) = &self.receipt {+ return if applied == insertion {+ Ok(*receipt)+ } else {+ Err(VoiceError::RevisionConflict)+ };+ }+ if !foreground || insertion.anchor.destination != destination {+ return Err(VoiceError::RevisionConflict);+ }+ if composing {+ return Err(VoiceError::PendingEdit);+ }+ if insertion.text.trim().is_empty() {+ return Err(VoiceError::InvalidState);+ }+ let mut anchor = insertion.anchor.clone();+ if anchor.revision != *revision {+ anchor.start_utf16 = text.encode_utf16().count() as u32;+ anchor.end_utf16 = anchor.start_utf16;+ }+ let (rendered, caret) = preview_insertion(text, &anchor, &insertion.text)?;+ *text = rendered;+ *revision += 1;+ self.receipt = Some((insertion.clone(), caret));+ Ok(caret)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn insertion(start: u32, end: u32, text: &str) -> VoiceInsertion {+ VoiceInsertion {+ text: text.into(),+ anchor: ComposerAnchor {+ destination: "chat".into(),+ revision: 7,+ start_utf16: start,+ end_utf16: end,+ },+ }+ }++ #[test]+ fn selected_text_is_replaced_once_even_after_lost_acknowledgement() {+ let mut owner = ComposerTransfer::default();+ let mut text = " meet Friday\n".to_owned();+ let mut revision = 7;+ let insert = insertion(7, 13, "Saturday");+ let receipt = owner+ .apply(&insert, "chat", &mut revision, &mut text, true, false)+ .unwrap();+ assert_eq!(text, " meet Saturday\n");+ assert_eq!(+ owner+ .apply(+ &insert,+ "different-chat",+ &mut revision,+ &mut text,+ false,+ false+ )+ .unwrap(),+ receipt+ );+ assert_eq!(revision, 8);+ assert_eq!(text, " meet Saturday\n");+ }++ #[test]+ fn changed_draft_appends_once_but_a_different_chat_is_rejected() {+ let mut owner = ComposerTransfer::default();+ let mut text = "My edited draft".to_owned();+ let mut revision = 8;+ let insert = insertion(0, 0, "hello");+ for (destination, foreground, composing) in [+ ("another-chat", true, false),+ ("chat", false, false),+ ("chat", true, true),+ ] {+ assert!(+ owner+ .apply(+ &insert,+ destination,+ &mut revision,+ &mut text,+ foreground,+ composing+ )+ .is_err()+ );+ }+ assert_eq!(text, "My edited draft");+ owner+ .apply(&insert, "chat", &mut revision, &mut text, true, false)+ .unwrap();+ assert_eq!(text, "My edited draft hello");+ owner+ .apply(&insert, "chat", &mut revision, &mut text, true, false)+ .unwrap();+ assert_eq!(text, "My edited draft hello");+ assert_eq!(revision, 9);+ }++ #[test]+ fn utf16_ranges_preserve_text_and_reject_split_surrogates() {+ for text in ["👩🏽💻", "e\u{301}", "कि", "क्ष"] {+ let end = text.encode_utf16().count() as u32;+ assert_eq!(utf16_range(text, 0, end).unwrap(), 0..text.len());+ for (byte, character) in text.char_indices() {+ let start = text[..byte].encode_utf16().count() as u32;+ assert_eq!(+ utf16_range(text, start, start + character.len_utf16() as u32).unwrap(),+ byte..byte + character.len_utf8()+ );+ if character.len_utf16() == 2 {+ assert_eq!(+ utf16_range(text, start, start + 1),+ Err(VoiceError::InvalidTextRange)+ );+ assert_eq!(+ utf16_range(text, start + 1, end),+ Err(VoiceError::InvalidTextRange)+ );+ }+ }+ }+ for (start, end) in [(2, 1), (0, 9), (9, 9)] {+ assert_eq!(+ utf16_range("hello", start, end),+ Err(VoiceError::InvalidTextRange)+ );+ }+ assert_eq!(utf16_range("你好 مرحبا", 0, 2).unwrap(), 0..6);+ }++ #[test]+ fn insertion_preserves_whitespace_and_script_boundaries() {+ assert_eq!(boundary_separator("hello", "world"), " ");+ assert_eq!(boundary_separator("你好", "世界"), "");+ assert_eq!(boundary_separator("hello\n", "world"), "");+ assert_eq!(boundary_separator("مرحبا", "بك"), " ");+ assert_eq!(boundary_separator("कि", "Saturday"), " ");+ assert_eq!(boundary_separator("क्", "Saturday"), " ");+ assert_eq!(boundary_separator("cafe\u{301}", "Saturday"), " ");+ assert_eq!(boundary_separator("你\u{301}", "好"), "");+ assert_eq!(boundary_separator("👩🏽💻", "Saturday"), "");+ assert_eq!(boundary_separator("Done.", "Next"), " ");+ assert_eq!(boundary_separator("है।", "बैठक"), " ");+ assert_eq!(boundary_separator("نعم؟", "موعد"), " ");+ assert_eq!(boundary_separator("안녕", "친구"), " ");+ }+}diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rsnew file mode 100644index 0000000000..dc70c90c0c--- /dev/null+++ b/rust/crates/ensu/src/voice/mod.rs@@ -0,0 +1,39 @@+mod audio;+mod capture;+mod composer;+mod rolling;+mod session;++pub use audio::*;+pub use capture::*;+pub use composer::*;+pub use rolling::*;+pub use session::*;++#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]+pub enum VoiceError {+ #[error("Audio continuity or format is invalid")]+ InvalidAudio,+ #[error("Audio backlog limit reached")]+ BacklogFull,+ #[error("Voice recording was cancelled")]+ Cancelled,+ #[error("Speech alignment could not be preserved")]+ UnstableTranscript,+ #[error("Speech processing failed")]+ ProcessingFailed,+ #[error("Voice event belongs to an inactive recording")]+ StaleEvent,+ #[error("Voice operation is not valid in the current state")]+ InvalidState,+ #[error("Text range is not on a character boundary")]+ InvalidTextRange,+ #[error("Text or destination changed")]+ RevisionConflict,+ #[error("Some speech could not be finalized")]+ IncompleteAudio,+ #[error("Keyboard input is still composing")]+ PendingEdit,+}++pub type Result<T> = std::result::Result<T, VoiceError>;diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rsnew file mode 100644index 0000000000..ba43201173--- /dev/null+++ b/rust/crates/ensu/src/voice/rolling.rs@@ -0,0 +1,511 @@+use std::collections::VecDeque;++use super::{+ AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, boundary_separator,+};++const PRE_SPEECH: u64 = VOICE_SAMPLE_RATE;+const PHRASE_SILENCE: u64 = VOICE_SAMPLE_RATE * 3;+const LOOKBACK: u64 = VOICE_SAMPLE_RATE * 3 / 2;+const UPDATE_SAMPLES: u64 = VOICE_SAMPLE_RATE / 2;+const ALIGNMENT_TOLERANCE: u64 = VOICE_SAMPLE_RATE / 5;+const REQUIRED_OBSERVATIONS: u8 = 3;+const RIGHT_CONTEXT_WORDS: usize = 2;++#[derive(Clone, Debug)]+pub struct AlignedWord {+ pub text: String,+ pub start: u64,+ pub end: u64,+}++pub struct DecodeWindow {+ pub id: u64,+ pub start: u64,+ pub end: u64,+ pub final_phrase: bool,+ pub samples: Vec<f32>,+}++#[derive(Debug)]+pub struct RecognitionUpdate {+ pub commit: Option<(u64, u64, String)>,+ pub hypothesis: String,+ pub covered_through: u64,+}++struct Region {+ start: u64,+ last_voice: u64,+ end: Option<u64>,+}++#[derive(Default)]+pub struct RollingRecognizer {+ ring: AudioRing,+ regions: VecDeque<Region>,+ previous: Vec<AlignedWord>,+ observations: Vec<u8>,+ anchor: Option<AlignedWord>,+ covered: u64,+ last_decoded: u64,+ next_id: u64,+ in_flight: Option<(u64, u64, u64, bool)>,+}++impl RollingRecognizer {+ pub fn feed(&mut self, samples: &[f32], voiced: bool) -> Result<()> {+ if samples.is_empty() {+ return Err(VoiceError::InvalidAudio);+ }+ let start = self.ring.accepted();+ self.ring.push(start, samples)?;+ let end = self.ring.accepted();+ if voiced {+ if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none()) {+ region.last_voice = end;+ } else {+ let prior_end = self+ .regions+ .back()+ .and_then(|r| r.end)+ .unwrap_or(self.covered);+ self.regions.push_back(Region {+ start: start+ .saturating_sub(PRE_SPEECH)+ .max(self.ring.retained_from())+ .max(prior_end),+ last_voice: end,+ end: None,+ });+ }+ } else if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none())+ && end - region.last_voice >= PHRASE_SILENCE+ {+ region.end = Some(end);+ }+ self.release_silence()?;+ Ok(())+ }++ fn release_silence(&mut self) -> Result<()> {+ if self.regions.is_empty() {+ let end = self.ring.accepted();+ self.ring.acknowledge(+ end,+ end.saturating_sub(PRE_SPEECH)+ .max(self.ring.retained_from()),+ )?;+ self.covered = self.covered.max(self.ring.retained_from());+ }+ Ok(())+ }++ pub fn finish(&mut self) {+ if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none()) {+ region.end = Some(self.ring.accepted());+ }+ }++ pub fn next_window(&mut self) -> Result<Option<DecodeWindow>> {+ if self.in_flight.is_some() {+ return Ok(None);+ }+ let Some(region) = self.regions.front() else {+ return Ok(None);+ };+ let end = region+ .end+ .unwrap_or(self.ring.accepted())+ .min(region.start + MAX_WINDOW_SAMPLES as u64);+ let final_phrase = region.end == Some(end);+ let at_limit = end - region.start == MAX_WINDOW_SAMPLES as u64;+ if !final_phrase+ && !at_limit+ && (end - region.start < UPDATE_SAMPLES || end < self.last_decoded + UPDATE_SAMPLES)+ {+ return Ok(None);+ }+ self.next_id += 1;+ self.in_flight = Some((self.next_id, region.start, end, final_phrase));+ Ok(Some(DecodeWindow {+ id: self.next_id,+ start: region.start,+ end,+ final_phrase,+ samples: self.ring.window(region.start, end)?,+ }))+ }++ pub fn complete(+ &mut self,+ request: &DecodeWindow,+ words: Vec<AlignedWord>,+ ) -> Result<RecognitionUpdate> {+ if self.in_flight != Some((request.id, request.start, request.end, request.final_phrase)) {+ return Err(VoiceError::StaleEvent);+ }+ let mut current = words;+ let mut previous_start = 0;+ for word in &mut current {+ if word.text.trim().is_empty()+ || word.start < previous_start+ || word.end < word.start+ || word.end > request.end - request.start+ {+ return Err(VoiceError::UnstableTranscript);+ }+ previous_start = word.start;+ word.start += request.start;+ word.end += request.start;+ }+ let skip = self.overlap_end(¤t)?;+ let remaining = ¤t[skip..];+ let old_skip = self.overlap_end(&self.previous)?;+ let mut observations = vec![1; current.len()];+ let agreed_prefix = remaining+ .iter()+ .zip(&self.previous[old_skip..])+ .take_while(|(a, b)| {+ a.text == b.text && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE+ })+ .count();+ for i in 0..agreed_prefix {+ observations[skip + i] = self.observations[old_skip + i].saturating_add(1);+ }+ let mature = observations[skip..]+ .iter()+ .take_while(|count| **count >= REQUIRED_OBSERVATIONS)+ .count()+ .saturating_sub(RIGHT_CONTEXT_WORDS);+ let agreed = remaining+ .iter()+ .take(mature)+ .take_while(|word| word.end <= request.end.saturating_sub(LOOKBACK))+ .count();+ let commit_count = if request.final_phrase {+ remaining.len()+ } else {+ agreed+ };+ if request.final_phrase && current.is_empty() {+ return Err(VoiceError::IncompleteAudio);+ }+ let commit = if commit_count > 0 {+ let end = if request.final_phrase {+ request.end+ } else {+ remaining[commit_count - 1].end+ };+ if end <= self.covered {+ return Err(VoiceError::UnstableTranscript);+ }+ let text = join_words(&remaining[..commit_count]);+ let start = self.covered.max(request.start);+ self.covered = end;+ self.anchor = Some(remaining[commit_count - 1].clone());+ Some((start, end, text))+ } else {+ None+ };+ let hypothesis = join_words(&remaining[commit_count..]);+ self.previous = current;+ self.observations = observations;+ self.last_decoded = request.end;+ self.in_flight = None;+ let mut retain_from = request.start;+ if request.final_phrase {+ self.covered = request.end;+ self.regions.pop_front();+ self.previous.clear();+ self.observations.clear();+ self.anchor = None;+ self.last_decoded = 0;+ retain_from = self.regions.front().map_or(request.end, |r| r.start);+ retain_from = retain_from.min(request.end);+ } else if request.end - request.start == MAX_WINDOW_SAMPLES as u64 {+ if self.covered <= request.start + LOOKBACK {+ return Err(VoiceError::UnstableTranscript);+ }+ retain_from = self+ .covered+ .saturating_sub(LOOKBACK)+ .min(self.anchor.as_ref().map_or(self.covered, |w| w.start))+ .max(request.start);+ if retain_from <= request.start {+ return Err(VoiceError::UnstableTranscript);+ }+ if let Some(region) = self.regions.front_mut() {+ region.start = retain_from;+ }+ }+ self.ring.acknowledge(+ request.end.max(self.ring.processed()),+ retain_from.max(self.ring.retained_from()),+ )?;+ self.release_silence()?;+ Ok(RecognitionUpdate {+ commit,+ hypothesis,+ covered_through: self.covered,+ })+ }++ fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {+ let Some(anchor) = &self.anchor else {+ return Ok(0);+ };+ if words.is_empty() {+ return Ok(0);+ }+ let anchor_key = alignment_key(&anchor.text);+ let mut matches = words.iter().enumerate().filter_map(|(i, word)| {+ (alignment_key(&word.text) == anchor_key+ && word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE)+ .then_some(i + 1)+ });+ match (matches.next(), matches.next()) {+ (Some(end), None) => Ok(end),+ (None, None) if words[0].start >= self.covered => Ok(0),+ _ => Err(VoiceError::UnstableTranscript),+ }+ }++ pub fn is_settled(&self) -> bool {+ self.regions.is_empty() && self.in_flight.is_none()+ }+ #[cfg(test)]+ fn covered(&self) -> u64 {+ self.covered+ }+ #[cfg(test)]+ fn retained_samples(&self) -> usize {+ self.ring.retained_samples()+ }+}++fn alignment_key(text: &str) -> String {+ let lexical = text.trim().trim_matches([+ '.', ',', '!', '?', ':', ';', '…', '。', ',', '!', '?', '؛', '،', '।', '॥', '¿', '¡',+ '"', '“', '”',+ ]);+ if lexical.is_empty() {+ text.trim().to_owned()+ } else {+ lexical.to_lowercase()+ }+}++fn join_words(words: &[AlignedWord]) -> String {+ let mut text = String::new();+ for word in words {+ text.push_str(boundary_separator(&text, &word.text));+ text.push_str(&word.text);+ }+ text+}++#[cfg(test)]+mod tests {+ use super::*;++ fn aligned(text: &str, start: u64, end: u64) -> AlignedWord {+ AlignedWord {+ text: text.into(),+ start,+ end,+ }+ }++ #[test]+ fn provisional_revisions_need_repeated_right_context_before_commit() {+ let mut rolling = RollingRecognizer::default();+ for index in 0..5 {+ rolling.feed(&vec![0.1; 8000], true).unwrap();+ let request = rolling.next_window().unwrap().unwrap();+ let first = if index < 2 { "Friday" } else { "Saturday" };+ let update = rolling+ .complete(+ &request,+ vec![+ aligned(first, 0, 1000),+ aligned("at", 1000, 2000),+ aligned("noon", 2000, 3000),+ ],+ )+ .unwrap();+ if index < 4 {+ assert!(update.commit.is_none());+ assert_eq!(update.hypothesis, format!("{first} at noon"));+ } else {+ assert_eq!(update.commit.unwrap().2, "Saturday");+ assert_eq!(update.hypothesis, "at noon");+ }+ }+ }++ #[test]+ fn repeated_unfinished_tail_stays_revisable_until_endpoint() {+ let mut rolling = RollingRecognizer::default();+ for _ in 0..6 {+ rolling.feed(&vec![0.1; 8000], true).unwrap();+ let request = rolling.next_window().unwrap().unwrap();+ let update = rolling+ .complete(+ &request,+ vec![aligned("This", 0, 1000), aligned("one.", 1000, 2000)],+ )+ .unwrap();+ assert!(update.commit.is_none());+ }+ rolling.finish();+ let request = rolling.next_window().unwrap().unwrap();+ let update = rolling+ .complete(+ &request,+ vec![aligned("And", 0, 1000), aligned("so", 1000, 2000)],+ )+ .unwrap();+ assert_eq!(update.commit.unwrap().2, "And so");+ assert!(rolling.is_settled());+ }++ #[test]+ fn short_vad_dropout_preserves_contiguous_audio_and_phrase() {+ let mut rolling = RollingRecognizer::default();+ rolling.feed(&vec![0.1; 8000], false).unwrap();+ rolling.feed(&vec![0.2; 8000], true).unwrap();+ rolling.feed(&vec![0.3; 16000], false).unwrap();+ rolling.feed(&vec![0.4; 8000], true).unwrap();+ let request = rolling.next_window().unwrap().unwrap();+ assert_eq!((request.start, request.end), (0, 40000));+ assert!(!request.final_phrase);+ assert_eq!(&request.samples[16000..32000], &vec![0.3; 16000]);+ assert_eq!(rolling.regions.len(), 1);+ }++ #[test]+ fn overlap_ignores_boundary_punctuation_but_preserves_acoustic_identity() {+ let rolling = RollingRecognizer {+ anchor: Some(aligned("You.", 16000, 20000)),+ covered: 20000,+ ..RollingRecognizer::default()+ };+ assert_eq!(+ rolling.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),+ Ok(1)+ );+ assert_eq!(+ rolling.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),+ Err(VoiceError::UnstableTranscript)+ );+ assert_eq!(+ rolling.overlap_end(&[aligned("you", 12000, 20000)]),+ Err(VoiceError::UnstableTranscript)+ );+ assert_eq!(+ rolling.overlap_end(&[aligned("not", 16000, 20000)]),+ Err(VoiceError::UnstableTranscript)+ );+ assert_ne!(alignment_key("don't"), alignment_key("do"));+ assert_ne!(alignment_key("कि"), alignment_key("क"));+ assert_ne!(alignment_key("-5"), alignment_key("5"));+ let rolling = RollingRecognizer {+ anchor: Some(aligned("Café!", 16000, 20000)),+ covered: 20000,+ ..RollingRecognizer::default()+ };+ assert_eq!(rolling.overlap_end(&[aligned("café", 16000, 20000)]), Ok(1));+ assert_eq!(+ rolling.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),+ Err(VoiceError::UnstableTranscript)+ );+ }++ fn words(request: &DecodeWindow) -> Vec<AlignedWord> {+ (0..request.end / 4000)+ .filter_map(|i| {+ let start = i * 4000;+ (start >= request.start).then(|| AlignedWord {+ text: "no".into(),+ start: start - request.start,+ end: (start + 3000).min(request.end) - request.start,+ })+ })+ .collect()+ }++ #[test]+ fn continuous_speech_advances_stable_text_without_losing_repetitions() {+ let mut rolling = RollingRecognizer::default();+ let mut committed = String::new();+ for _ in 0..7200 {+ rolling.feed(&vec![0.2; 4000], true).unwrap();+ if let Some(request) = rolling.next_window().unwrap() {+ assert!(request.samples.len() <= MAX_WINDOW_SAMPLES);+ let update = rolling.complete(&request, words(&request)).unwrap();+ if let Some((_, _, text)) = update.commit {+ committed.push_str(&text);+ committed.push(' ');+ }+ }+ assert!(rolling.retained_samples() <= 30 * 16000);+ }+ assert!(committed.split_whitespace().count() > 7100);+ rolling.finish();+ while let Some(request) = rolling.next_window().unwrap() {+ if let Some((_, _, text)) = rolling.complete(&request, words(&request)).unwrap().commit+ {+ committed.push_str(&text);+ committed.push(' ');+ }+ }+ assert!(rolling.is_settled());+ assert_eq!(committed.split_whitespace().count(), 7200);+ }++ #[test]+ fn final_short_speech_and_stale_results_are_handled() {+ let mut rolling = RollingRecognizer::default();+ rolling.feed(&vec![0.1; 3200], true).unwrap();+ assert!(rolling.next_window().unwrap().is_none());+ rolling.finish();+ let request = rolling.next_window().unwrap().unwrap();+ let update = rolling+ .complete(+ &request,+ vec![AlignedWord {+ text: "Ha".into(),+ start: 0,+ end: 3200,+ }],+ )+ .unwrap();+ assert_eq!(update.commit.unwrap().2, "Ha");+ assert!(matches!(+ rolling.complete(&request, Vec::new()),+ Err(VoiceError::StaleEvent)+ ));+ }++ #[test]+ fn failed_alignment_keeps_audio_and_does_not_claim_completion() {+ let mut rolling = RollingRecognizer::default();+ rolling.feed(&vec![0.1; 8000], true).unwrap();+ let request = rolling.next_window().unwrap().unwrap();+ assert!(+ rolling+ .complete(+ &request,+ vec![AlignedWord {+ text: "bad".into(),+ start: 0,+ end: 9000+ }]+ )+ .is_err()+ );+ assert_eq!(rolling.retained_samples(), 8000);+ assert_eq!(rolling.covered(), 0);+ }+}diff --git a/rust/crates/ensu/src/voice/session.rs b/rust/crates/ensu/src/voice/session.rsnew file mode 100644index 0000000000..613015d145--- /dev/null+++ b/rust/crates/ensu/src/voice/session.rs@@ -0,0 +1,332 @@+use super::{Result, VOICE_SAMPLE_RATE, VoiceError};++const SILENCE_SAMPLES: u64 = 3 * VOICE_SAMPLE_RATE;+const COUNTDOWN_SAMPLES: u64 = VOICE_SAMPLE_RATE * 3 / 5;+const NO_SPEECH_SAMPLES: u64 = 15 * VOICE_SAMPLE_RATE;+const SESSION_LIMIT_SAMPLES: u64 = 30 * 60 * VOICE_SAMPLE_RATE;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum VoicePhase {+ Preparing,+ Listening,+ Countdown,+ Finishing,+ Ready,+ Closed,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum StopReason {+ Done,+ Silence,+ Interruption,+ Failure,+ Limit,+ NoSpeech,+ Discard,+}++#[derive(Debug)]+pub struct VoiceSession {+ phase: VoicePhase,+ accepted: u64,+ recognized: u64,+ waits_for_live_text: bool,+ last_voice: Option<u64>,+ stop_reason: Option<StopReason>,+ worker_active: bool,+ incomplete: bool,+}++impl Default for VoiceSession {+ fn default() -> Self {+ Self {+ phase: VoicePhase::Preparing,+ accepted: 0,+ recognized: 0,+ waits_for_live_text: true,+ last_voice: None,+ stop_reason: None,+ worker_active: false,+ incomplete: false,+ }+ }+}++impl VoiceSession {+ pub fn recorded() -> Self {+ Self {+ waits_for_live_text: false,+ ..Self::default()+ }+ }+ pub fn phase(&self) -> VoicePhase {+ self.phase+ }+ pub fn accepted_samples(&self) -> u64 {+ self.accepted+ }+ pub fn worker_active(&self) -> bool {+ self.worker_active+ }+ pub fn incomplete(&self) -> bool {+ self.incomplete+ }+ pub fn stop_reason(&self) -> Option<StopReason> {+ self.stop_reason+ }++ pub fn begin_capture(&mut self) -> Result<()> {+ if self.phase != VoicePhase::Preparing {+ return Err(VoiceError::InvalidState);+ }+ self.worker_active = true;+ self.phase = VoicePhase::Listening;+ Ok(())+ }++ pub fn defer_recognition(&mut self) {+ self.waits_for_live_text = false;+ self.update_countdown();+ }++ pub fn observe_audio(+ &mut self,+ start: u64,+ end: u64,+ voiced_end: Option<u64>,+ ) -> Result<Option<StopReason>> {+ if !matches!(self.phase, VoicePhase::Listening | VoicePhase::Countdown) {+ return Err(VoiceError::InvalidState);+ }+ if start != self.accepted+ || end <= start+ || voiced_end.is_some_and(|v| v <= start || v > end)+ {+ return Err(VoiceError::InvalidAudio);+ }+ self.accepted = end;+ if let Some(voice) = voiced_end {+ self.last_voice = Some(voice);+ self.phase = VoicePhase::Listening;+ }+ if end >= SESSION_LIMIT_SAMPLES {+ return Ok(Some(StopReason::Limit));+ }+ if self.last_voice.is_none() && end >= NO_SPEECH_SAMPLES {+ return Ok(Some(StopReason::NoSpeech));+ }+ self.update_countdown();+ Ok(None)+ }++ pub fn observe_recognition(&mut self, has_text: bool) -> Result<()> {+ if !self.worker_active || self.phase == VoicePhase::Closed {+ return Err(VoiceError::InvalidState);+ }+ if has_text && self.waits_for_live_text {+ self.waits_for_live_text = false;+ self.update_countdown();+ }+ Ok(())+ }++ fn silence_deadline(&self) -> Option<u64> {+ self.last_voice.map(|voice| {+ let deadline = voice + SILENCE_SAMPLES;+ if !self.waits_for_live_text {+ deadline+ } else {+ deadline.max(NO_SPEECH_SAMPLES)+ }+ })+ }+ fn update_countdown(&mut self) {+ if matches!(self.phase, VoicePhase::Listening | VoicePhase::Countdown) {+ self.phase = if self.silence_deadline().is_some_and(|deadline| {+ self.accepted >= deadline - (SILENCE_SAMPLES - COUNTDOWN_SAMPLES)+ }) {+ VoicePhase::Countdown+ } else {+ VoicePhase::Listening+ };+ }+ }+ pub fn silence_elapsed(&self) -> bool {+ self.phase == VoicePhase::Countdown+ && self+ .silence_deadline()+ .is_some_and(|deadline| self.accepted >= deadline)+ }+ pub fn stop(&mut self, reason: StopReason) -> Result<bool> {+ if self.phase == VoicePhase::Closed {+ return Ok(false);+ }+ if reason == StopReason::Discard {+ self.stop_reason = Some(reason);+ self.phase = VoicePhase::Closed;+ return Ok(true);+ }+ if !self.worker_active {+ return Err(VoiceError::InvalidState);+ }+ let first = self.stop_reason.is_none();+ if first || matches!(reason, StopReason::Interruption | StopReason::Failure) {+ self.stop_reason = Some(reason);+ self.phase = VoicePhase::Finishing;+ }+ Ok(first)+ }+ pub fn recognize_through(&mut self, end: u64) -> Result<()> {+ if !self.worker_active {+ return Err(VoiceError::InvalidState);+ }+ if end < self.recognized || end > self.accepted {+ return Err(VoiceError::InvalidAudio);+ }+ self.recognized = end;+ Ok(())+ }+ pub fn observe_drained_audio(&mut self, start: u64, end: u64) -> Result<()> {+ if !self.worker_active || self.phase == VoicePhase::Closed || self.stop_reason.is_none() {+ return Err(VoiceError::InvalidState);+ }+ if start != self.accepted || end <= start {+ return Err(VoiceError::InvalidAudio);+ }+ self.accepted = end;+ Ok(())+ }+ pub fn worker_exited(&mut self, successful: bool) -> Result<()> {+ if !self.worker_active {+ return Err(VoiceError::InvalidState);+ }+ let reason = self.stop_reason.ok_or(VoiceError::InvalidState)?;+ self.worker_active = false;+ self.incomplete = !successful+ || self.recognized < self.accepted+ || matches!(reason, StopReason::Interruption | StopReason::Failure);+ if self.phase != VoicePhase::Closed {+ self.phase = VoicePhase::Ready;+ }+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn speaking() -> VoiceSession {+ let mut session = VoiceSession::default();+ session.begin_capture().unwrap();+ session.observe_audio(0, 16000, Some(16000)).unwrap();+ session.observe_recognition(true).unwrap();+ session+ }++ #[test]+ fn silence_uses_the_same_audio_clock_after_live_or_deferred_recognition() {+ for (mut session, deferred) in [+ (speaking(), false),+ (VoiceSession::recorded(), false),+ (VoiceSession::default(), true),+ ] {+ if session.phase() == VoicePhase::Preparing {+ session.begin_capture().unwrap();+ session.observe_audio(0, 16000, Some(16000)).unwrap();+ }+ if deferred {+ session.defer_recognition();+ }+ session.observe_audio(16000, 32000, None).unwrap();+ assert!(!session.silence_elapsed());+ session.observe_audio(32000, 64000, None).unwrap();+ assert!(session.silence_elapsed());+ assert!(session.stop(StopReason::Silence).unwrap());+ assert!(!session.stop(StopReason::Done).unwrap());+ assert!(session.worker_active());+ assert_eq!(session.begin_capture(), Err(VoiceError::InvalidState));+ session.recognize_through(64000).unwrap();+ session.worker_exited(true).unwrap();+ assert_eq!(session.phase(), VoicePhase::Ready);+ assert!(!session.incomplete());+ }+ }++ #[test]+ fn live_grace_waits_for_text_or_fifteen_seconds() {+ let mut session = VoiceSession::default();+ session.begin_capture().unwrap();+ session.observe_audio(0, 32000, Some(24000)).unwrap();+ session.observe_recognition(false).unwrap();+ session.observe_audio(32000, 80000, None).unwrap();+ assert_eq!(session.phase(), VoicePhase::Listening);+ session.observe_audio(80000, 13 * 16000, None).unwrap();+ assert!(!session.silence_elapsed());+ assert_eq!(session.silence_deadline(), Some(15 * 16000));+ session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();+ assert!(session.silence_elapsed());+ }++ #[test]+ fn recognition_and_later_speech_recompute_the_current_deadline() {+ let mut session = VoiceSession::default();+ session.begin_capture().unwrap();+ session.observe_audio(0, 16000, Some(16000)).unwrap();+ session.observe_audio(16000, 13 * 16000, None).unwrap();+ assert!(!session.silence_elapsed());+ session.observe_recognition(true).unwrap();+ assert!(session.silence_elapsed());+ session+ .observe_audio(13 * 16000, 14 * 16000, Some(14 * 16000))+ .unwrap();+ assert!(!session.silence_elapsed());+ assert_eq!(session.phase(), VoicePhase::Listening);+ session.observe_audio(14 * 16000, 16 * 16000, None).unwrap();+ assert!(!session.silence_elapsed());+ session.observe_audio(16 * 16000, 17 * 16000, None).unwrap();+ assert!(session.silence_elapsed());+ }++ #[test]+ fn silence_and_duration_limits_apply_to_both_tiers() {+ for mut session in [VoiceSession::default(), VoiceSession::recorded()] {+ session.begin_capture().unwrap();+ assert_eq!(+ session.observe_audio(0, NO_SPEECH_SAMPLES, None).unwrap(),+ Some(StopReason::NoSpeech)+ );+ assert!(!session.silence_elapsed());+ assert_eq!(+ session+ .observe_audio(+ NO_SPEECH_SAMPLES,+ SESSION_LIMIT_SAMPLES,+ Some(SESSION_LIMIT_SAMPLES)+ )+ .unwrap(),+ Some(StopReason::Limit)+ );+ }+ }++ #[test]+ fn interruption_retains_ownership_until_exit_and_cannot_resume() {+ let mut session = speaking();+ session.stop(StopReason::Done).unwrap();+ session.stop(StopReason::Interruption).unwrap();+ assert!(session.worker_active());+ assert_eq!(session.phase(), VoicePhase::Finishing);+ session.observe_drained_audio(16000, 32000).unwrap();+ session.worker_exited(false).unwrap();+ assert!(session.incomplete());+ assert_eq!(session.phase(), VoicePhase::Ready);+ assert_eq!(session.begin_capture(), Err(VoiceError::InvalidState));+ let mut fresh = VoiceSession::default();+ fresh.begin_capture().unwrap();+ fresh.observe_audio(0, 16000, Some(16000)).unwrap();+ fresh.observe_audio(16000, 80000, None).unwrap();+ assert!(!fresh.silence_elapsed());+ }+}