💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Finalize mobile dictation recovery and controls
setalp7fbc4076c250
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 537b82b3f6..149067b9ff 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -92,6 +92,15 @@ class AppStore(private var composerRevision = 0uLprivate var emptyComposerId = UUID.randomUUID().toString()+ private var composerSelectionStart = 0+ private var composerSelectionEnd = 0+ private var composerIsComposing = false++ internal fun updateComposerSelection(start: Int, end: Int, isComposing: Boolean) {+ composerSelectionStart = start+ composerSelectionEnd = end+ composerIsComposing = isComposing+ }private fun invalidateComposer() {composerRevision++@@ -138,14 +147,30 @@ class AppStore(if (selectionStart !in 0..selectionEnd || selectionEnd > text.length) returnval characters = BreakIterator.getCharacterInstance(Locale.ROOT).apply { setText(text) }if (!characters.isBoundary(selectionStart) || !characters.isBoundary(selectionEnd)) return- voiceController.open(- DictationInsertion(- chat.currentSessionId ?: emptyComposerId,- composerRevision,- selectionStart.toUInt(),- selectionEnd.toUInt(),+ updateComposerSelection(selectionStart, selectionEnd, isComposing)+ val destination = chat.currentSessionId ?: emptyComposerId+ voiceController.open {+ val current = _state.value.chat+ val start = composerSelectionStart+ val end = composerSelectionEnd+ val boundaries =+ BreakIterator.getCharacterInstance(Locale.ROOT).apply {+ setText(current.messageText)+ }+ if (+ destination != (current.currentSessionId ?: emptyComposerId) ||+ current.isGenerating ||+ current.isDownloading ||+ current.editingMessageId != null ||+ composerIsComposing ||+ start !in 0..end ||+ end > current.messageText.length ||+ !boundaries.isBoundary(start) ||+ !boundaries.isBoundary(end))- )+ null+ else DictationInsertion(destination, composerRevision, start.toUInt(), end.toUInt())+ }}fun dispose() = voiceController.dispose()@@ -171,7 +196,10 @@ class AppStore(modelSettingsActions.setAppForeground(active)}- fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)+ fun setPickerPending(pending: Boolean) {+ if (pending) voiceController.discard(keepDownload = true)+ modelSettingsActions.setPickerPending(pending)+ }fun handleMemoryPressure() {voiceController.interrupt(VoiceInputFailure.MemoryUnavailable)@@ -246,27 +274,27 @@ class AppStore(}fun createNewSession(): String {- voiceController.discard()+ voiceController.discard(keepDownload = true)invalidateComposer()return chatActions.createNewSession()}fun startNewSessionDraft() {- voiceController.discard()+ voiceController.discard(keepDownload = true)emptyComposerId = UUID.randomUUID().toString()invalidateComposer()chatActions.startNewSessionDraft()}fun selectSession(sessionId: String) {- voiceController.discard()+ voiceController.discard(keepDownload = true)invalidateComposer()chatActions.selectSession(sessionId)}fun deleteSession(sessionId: String) {if (sessionId == _state.value.chat.currentSessionId) {- voiceController.discard()+ voiceController.discard(keepDownload = true)invalidateComposer()}chatActions.deleteSession(sessionId)@@ -286,7 +314,7 @@ class AppStore(chatActions.updateBranchSelection(messageId, selectedIndex)fun beginEditing(messageId: String) {- voiceController.discard()+ voiceController.discard(keepDownload = true)invalidateComposer()chatActions.beginEditing(messageId)}@@ -335,10 +363,14 @@ class AppStore(modelSettingsActions.cancelModelDownload()}- fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)+ fun retryAssistantMessage(messageId: String) {+ voiceController.interrupt()+ chatActions.retryAssistantMessage(messageId)+ }fun sendMessage() {- if (voiceController.active) return+ if (voiceController.blocksComposer) return+ voiceController.discard(keepDownload = true)invalidateComposer()chatActions.sendMessage()}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex b306bf813e..f03aae6ee8 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -128,6 +128,7 @@ internal fun HomeNavigation(chatState = appState.chat,isDrawerOpen = drawerState.currentValue == DrawerValue.Open,onMessageChange = store::updateMessageText,+ onComposerSelection = store::updateComposerSelection,onSend = store::sendMessage,onStop = store::stopGeneration,onCancelDownload = store::cancelDownload,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.ktindex 9c3938b402..8c7f62065f 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt@@ -39,6 +39,7 @@ import androidx.compose.ui.focus.FocusRequesterimport androidx.compose.ui.focus.focusRequesterimport androidx.compose.ui.graphics.SolidColorimport androidx.compose.ui.hapticfeedback.HapticFeedbackType+import androidx.compose.ui.platform.LocalFocusManagerimport androidx.compose.ui.res.painterResourceimport androidx.compose.ui.text.TextRangeimport androidx.compose.ui.text.input.KeyboardCapitalization@@ -69,6 +70,7 @@ internal fun MessageInput(sourcesError: String?,onRetrySources: () -> Unit,onMessageChange: (String) -> Unit,+ onComposerSelection: (Int, Int, Boolean) -> Unit,onSend: () -> Unit,onStop: () -> Unit,onAttachmentSelected: (AttachmentType) -> Unit,@@ -80,6 +82,7 @@ internal fun MessageInput(focusRequestId: Int,) {val haptic = rememberHaptics()+ val focusManager = LocalFocusManager.currentval placeholder = "Ask Ensu"val composerIconSize = 18.dp@@ -93,7 +96,7 @@ internal fun MessageInput()}- LaunchedEffect(messageText, voiceCaret, voiceController.active) {+ LaunchedEffect(messageText, voiceCaret, voiceController.blocksComposer) {if (messageText != fieldValue.text || voiceCaret != null) {fieldValue =TextFieldValue(@@ -104,6 +107,13 @@ internal fun MessageInput(),)}+ if (!voiceController.blocksComposer) {+ onComposerSelection(+ fieldValue.selection.min,+ fieldValue.selection.max,+ fieldValue.composition != null,+ )+ }}LaunchedEffect(focusRequestId) {@@ -218,7 +228,7 @@ internal fun MessageInput(modifier = Modifier.fillMaxWidth(),verticalAlignment = Alignment.CenterVertically,) {- if (supportsImageAttachments && !voiceController.active) {+ if (supportsImageAttachments && !voiceController.blocksComposer) {IconButton(onClick = {haptic.perform(HapticFeedbackType.TextHandleMove)@@ -246,14 +256,21 @@ internal fun MessageInput(value = fieldValue,onValueChange = { newValue ->fieldValue = newValue- if (!voiceController.active) onMessageChange(newValue.text)+ if (!voiceController.blocksComposer) {+ onMessageChange(newValue.text)+ onComposerSelection(+ newValue.selection.min,+ newValue.selection.max,+ newValue.composition != null,+ )+ }},modifier = Modifier.weight(1f).focusRequester(focusRequester),textStyle = EnsuTypography.message.copy(color = EnsuColor.textPrimary()),visualTransformation = LongMessageVisualTransformation,minLines = 1,- maxLines = if (voiceController.active) 8 else 5,- readOnly = voiceController.active,+ maxLines = if (voiceController.blocksComposer) 8 else 5,+ readOnly = voiceController.blocksComposer,keyboardOptions =KeyboardOptions(capitalization = KeyboardCapitalization.Sentences),cursorBrush = SolidColor(EnsuColor.accent()),@@ -276,9 +293,17 @@ internal fun MessageInput(if (editingMessage == null) {VoiceInputButton(controller = voiceController,- enabled = !isGenerating && !isDownloading && fieldValue.composition == null,+ enabled = !isGenerating && !isDownloading,onOpen = {haptic.perform(HapticFeedbackType.TextHandleMove)+ focusManager.clearFocus(force = true)+ fieldValue = fieldValue.copy(composition = null)+ onMessageChange(fieldValue.text)+ onComposerSelection(+ fieldValue.selection.min,+ fieldValue.selection.max,+ false,+ )onVoiceInput(fieldValue)},)@@ -287,7 +312,7 @@ internal fun MessageInput(val canSend =sourcesReady &&(messageText.isNotBlank() || attachments.isNotEmpty()) &&- !voiceController.active &&+ !voiceController.blocksComposer &&!hasUnsupportedImagesIconButton(diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex 6b0d17dca9..b83f6be40e 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -947,9 +947,6 @@ internal class ChatStoreActions(systemPrompt = normalSystemPrompt,)- if (normalHistorySelection.currentInputTooLarge) {- throw ConversationException.CurrentInputTooLarge()- }if (normalHistorySelection.wasTrimmed &&overflowBypassMessageId != userMessage.id@@ -1089,18 +1086,6 @@ internal class ChatStoreActions(throw err} catch (_: LlmException.Cancelled) {interrupted = true- } catch (_: ConversationException.CurrentInputTooLarge) {- if (isActive() && !stopRequested) {- state.update {- it.copy(- chat =- it.chat.copy(- transientAssistantError = CURRENT_INPUT_TOO_LARGE_MESSAGE,- transientAssistantParentId = userMessage.id,- )- )- }- }} catch (err: ModelMemoryUnavailable) {if (isActive() && !stopRequested) {state.update {@@ -1861,7 +1846,6 @@ internal class ChatStoreActions(val inputTokens: Int,val inputBudget: Int,val wasTrimmed: Boolean,- val currentInputTooLarge: Boolean,)private data class GenerationLimits(@@ -1933,7 +1917,6 @@ internal class ChatStoreActions(inputTokens,inputBudget,inputTokens > inputBudget,- systemTokens + promptTokens > inputBudget,)}@@ -1964,13 +1947,7 @@ internal class ChatStoreActions()}- return HistorySelection(- selected,- inputTokens,- inputBudget,- inputTokens > inputBudget,- false,- )+ return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)}private fun resolveGenerationLimits(contextLength: Int): GenerationLimits {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex 9bce45e391..5f929cef14 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -50,6 +50,7 @@ internal fun ChatView(onOpenVoiceInput: (androidx.compose.ui.text.input.TextFieldValue) -> Unit,isDrawerOpen: Boolean,onMessageChange: (String) -> Unit,+ onComposerSelection: (Int, Int, Boolean) -> Unit,onSend: () -> Unit,onStop: () -> Unit,onCancelDownload: () -> Unit,@@ -235,7 +236,7 @@ internal fun ChatView(inputBarHeightDp = with(density) { coords.size.height.toDp() }})- } else if (chatState.isModelStateKnown) {+ } else if (chatState.isModelStateKnown && !showDownloadOnboarding) {val voicePreview = voiceController.composerPreview(chatState.messageText)MessageInput(modifier =@@ -256,6 +257,7 @@ internal fun ChatView(sourcesError = chatState.sourceSelectionError,onRetrySources = onRetrySources,onMessageChange = onMessageChange,+ onComposerSelection = onComposerSelection,onSend = {focusManager.clearFocus()onSend()@@ -265,10 +267,7 @@ internal fun ChatView(onRemoveAttachment = onRemoveAttachment,onCancelEdit = onCancelEdit,voiceController = voiceController,- onVoiceInput = { value ->- onOpenVoiceInput(value)- focusManager.clearFocus()- },+ onVoiceInput = onOpenVoiceInput,voiceCaret = voicePreview?.caretUtf16?.toInt() ?: voiceController.composerCaret,focusRequestId = focusRequestId,)diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex 55a0c40e4e..352571fc92 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -57,6 +57,7 @@ internal enum class VoiceInputFailure(val message: String) {MemoryUnavailable("Not enough free memory. Close other apps and try again."),MicrophoneUnavailable("Could not start the microphone. Tap the microphone to try again."),Interrupted("Voice input was interrupted. Review the available text and try again."),+ Incomplete("Some words may be missing. Review the text."),ProcessingFailed("Could not transcribe voice input. Review the available text and try again."),}@@ -84,7 +85,7 @@ internal sealed interface VoiceInputState {data object Recording : VoiceInputState- data class Downloading(val percent: Int?) : VoiceInputState+ data class Downloading(val percent: Int?, val queued: Boolean = true) : VoiceInputStatedata object Transcribing : VoiceInputState@@ -130,8 +131,10 @@ internal class VoiceTranscriptionController(private var capture: LiveMicrophone? = nullprivate var work: Job? = nullprivate var finishingNoticeJob: Job? = null+ private var transientErrorJob: Job? = nullprivate var attemptFinished = falseprivate var insertion: DictationInsertion? = null+ private var insertionFactory: (() -> DictationInsertion?)? = nullprivate var foreground = trueprivate var generation = 0Lval sessionVersion: Long@@ -166,13 +169,17 @@ internal class VoiceTranscriptionController(val canOpen: Booleanget() = !active && !hasActiveWork- fun open(insertion: DictationInsertion) {+ val blocksComposer: Boolean+ get() = active && state != VoiceInputState.Idle && state !is VoiceInputState.Downloading++ fun open(insertionFactory: () -> DictationInsertion?) {if (!canOpen) return+ transientErrorJob?.cancel()generation++initialRecordPending = trueattemptFinished = falseisFinishingSlow = false- this.insertion = insertion+ this.insertionFactory = insertionFactorysession = DictationSession(live = usesLiveDictation)snapshot = session?.snapshot()failure = null@@ -183,7 +190,7 @@ internal class VoiceTranscriptionController(fun composerPreview(original: String): io.ente.ensu.bindings.DictationInsertionResult? {val value = snapshot ?: return null- if (!active) return null+ if (!blocksComposer) return nullreturn runCatching { insertion?.preview(original, value.text + value.hypothesis) }.getOrNull()}@@ -211,14 +218,17 @@ internal class VoiceTranscriptionController(startOnForeground = falserecord()} else session?.let { refresh(it, generation) }- } else if (!awaitingPermission && !startOnForeground) interrupt()+ } else if (+ !awaitingPermission && !startOnForeground && state !is VoiceInputState.Downloading+ )+ interrupt()}fun record() {val owner = session ?: returnif (!active || !foreground || hasActiveWork || attemptFinished) returnfailure = null- state = VoiceInputState.Preparing+ state = VoiceInputState.Downloading(null)val token = generationhasActiveWork = truework =@@ -232,16 +242,30 @@ internal class VoiceTranscriptionController(modelAssets.all { assetStore.isDownloaded(it) }}) {- state = VoiceInputState.Downloading(null)assetStore.download(modelAssets) { progress ->scope.launch {if (token == generation && state is VoiceInputState.Downloading)state =- VoiceInputState.Downloading(progress.percentage.toInt())+ VoiceInputState.Downloading(+ progress.totalBytes?.let {+ progress.percentage.toInt()+ },+ progress.queued,+ )}}}- if (token != generation || !foreground) return@launch+ if (token != generation) return@launch+ if (!foreground) {+ discard()+ return@launch+ }+ insertion = insertionFactory?.invoke()+ insertionFactory = null+ if (insertion == null) {+ discard()+ return@launch+ }state = VoiceInputState.Preparingval activeStream = DictationStream(owner, 16000u, 1u)stream = activeStream@@ -370,6 +394,8 @@ internal class VoiceTranscriptionController(val completedState =when {added && !value.incomplete -> VoiceInputState.Idle+ added && failure == VoiceInputFailure.ProcessingFailed ->+ VoiceInputState.Error(VoiceInputFailure.Incomplete)text.isBlank() -> VoiceInputState.Error(failure ?: VoiceInputFailure.NoSpeech)else -> VoiceInputState.Error(failure ?: VoiceInputFailure.Interrupted)}@@ -378,8 +404,15 @@ internal class VoiceTranscriptionController(snapshot = nullsession = nullinsertion = null+ insertionFactory = nullaudioLevels = List(24) { 0f }state = completedState+ if (completedState == VoiceInputState.Error(VoiceInputFailure.NoSpeech)) {+ transientErrorJob = scope.launch {+ delay(10_000)+ if (token == generation && state == completedState) state = VoiceInputState.Idle+ }+ }}private fun watchFinalization(@@ -426,6 +459,10 @@ internal class VoiceTranscriptionController(fun interrupt(reason: VoiceInputFailure = VoiceInputFailure.Interrupted) {if (!active || attemptFinished) return+ if (state is VoiceInputState.Downloading) {+ insertionFactory = null+ return+ }if (!hasActiveWork && stream == null) {discard()return@@ -439,10 +476,16 @@ internal class VoiceTranscriptionController(isFinishingSlow = false}- fun discard() {+ fun discard(keepDownload: Boolean = false) {+ if (keepDownload && state is VoiceInputState.Downloading && hasActiveWork) {+ insertionFactory = null+ return+ }awaitingPermission = falsestartOnForeground = falsegeneration+++ transientErrorJob?.cancel()+ transientErrorJob = nullfinishingNoticeJob?.cancel()finishingNoticeJob = nullcapture?.stop()@@ -454,6 +497,7 @@ internal class VoiceTranscriptionController(snapshot = nullsession = nullinsertion = null+ insertionFactory = nullstream = nullcapture = null}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.ktindex dd79663204..f7e40398b3 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt@@ -38,6 +38,7 @@ import androidx.compose.ui.Modifierimport androidx.compose.ui.geometry.CornerRadiusimport androidx.compose.ui.geometry.Offsetimport androidx.compose.ui.geometry.Size+import androidx.compose.ui.hapticfeedback.HapticFeedbackTypeimport androidx.compose.ui.platform.LocalContextimport androidx.compose.ui.res.painterResourceimport androidx.compose.ui.semantics.LiveRegionMode@@ -51,6 +52,7 @@ import io.ente.ensu.designsystem.EnsuColorimport io.ente.ensu.designsystem.EnsuSpacingimport io.ente.ensu.designsystem.EnsuTypographyimport io.ente.ensu.designsystem.HugeIcons+import io.ente.ensu.platform.rememberHaptics@Composableinternal fun VoiceInputControls(controller: VoiceTranscriptionController) {@@ -60,7 +62,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {val context = LocalContext.currentval label =when (state) {- is VoiceInputState.Downloading -> "Downloading"+ is VoiceInputState.Downloading -> if (state.queued) "Download queued" else "Downloading"VoiceInputState.Preparing,VoiceInputState.Idle -> "Preparing"VoiceInputState.Transcribing ->@@ -98,7 +100,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))else Spacer(Modifier.weight(1f))if (controller.active && !state.isRecording) {- TextButton(onClick = controller::discard) { Text("Cancel") }+ TextButton(onClick = { controller.discard() }) { Text("Cancel") }}}}@@ -110,6 +112,7 @@ internal fun VoiceInputButton(onOpen: () -> Unit,) {val context = LocalContext.current+ val haptic = rememberHaptics()var permissionRequest by rememberSaveable { mutableStateOf<Long?>(null) }val permission =rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {@@ -140,6 +143,7 @@ internal fun VoiceInputButton(val canPress = if (controller.active) capturing else enabled && controller.canOpenval description =when {+ downloading?.queued == true -> "Voice model download queued"downloading != null -> "Downloading voice model"preparing -> "Preparing voice input"transcribing -> "Transcribing"@@ -149,17 +153,20 @@ internal fun VoiceInputButton(IconButton(onClick = {when {- capturing -> controller.stop()+ capturing -> {+ haptic.perform(HapticFeedbackType.LongPress)+ controller.stop()+ }else -> onOpen()}},enabled = canPress,- modifier = Modifier.size(44.dp).semantics { contentDescription = description },+ modifier = Modifier.size(36.dp).semantics { contentDescription = description },) {when {downloading != null || preparing || transcribing ->Box(Modifier.size(28.dp), contentAlignment = Alignment.Center) {- if (downloading?.percent != null)+ if (downloading?.percent != null && !downloading.queued)CircularProgressIndicator(progress = { downloading.percent.coerceIn(0, 100) / 100f },modifier = Modifier.size(28.dp),@@ -187,7 +194,7 @@ internal fun VoiceInputButton(tint = EnsuColor.backgroundBase(),)}- else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(20.dp))+ else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(18.dp))}}}diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swiftindex c4ada28a33..be96b716a0 100644--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift@@ -364,7 +364,7 @@ struct ChatView: View {}.padding(.bottom, keyboard.isVisible ? keyboard.height : 0).zIndex(1)- } else {+ } else if viewModel.isModelDownloaded {let shouldAutoFocus = shouldAutoFocusInputMessageInputView(diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex fe2100e4fd..97325b6174 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -534,21 +534,9 @@ final class ChatViewModel: ObservableObject {guard !isGenerating, !isDownloading, !isSendPending, editingMessageId == nil,!composerIsComposing, !voiceTranscriber.active, !voiceTranscriber.hasActiveTaskselse { return }- let range = composerSelection- let text = draftText as NSString- guard range.location >= 0, range.length >= 0, range.location <= text.length,- range.length <= text.length - range.location,- [range.location, NSMaxRange(range)].allSatisfy({- $0 == text.length || text.rangeOfComposedCharacterSequence(at: $0).location == $0- })- else { return }sessionSummaryTask?.cancel()cancelChatWarmup(releaseLoadedModel: false)let destination = (currentSessionId ?? emptyComposerId).uuidString- let revision = composerRevision- let insertion = DictationInsertion(- destination: destination, revision: revision,- startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))voiceTranscriber.onInsert = { [weak self] insertion, text inguard let self, self.appForeground else { return false }do {@@ -563,11 +551,29 @@ final class ChatViewModel: ObservableObject {return true} catch { return false }}- voiceTranscriber.open(insertion: insertion)+ voiceTranscriber.open { [weak self] in+ guard let self, self.appForeground, self.chatActive,+ !self.isGenerating, !self.isDownloading, !self.isSendPending,+ self.editingMessageId == nil, !self.composerIsComposing,+ (self.currentSessionId ?? self.emptyComposerId).uuidString == destination+ else { return nil }+ let range = self.composerSelection+ let text = self.draftText as NSString+ guard range.location >= 0, range.length >= 0, range.location <= text.length,+ range.length <= text.length - range.location,+ [range.location, NSMaxRange(range)].allSatisfy({+ $0 == text.length+ || text.rangeOfComposedCharacterSequence(at: $0).location == $0+ })+ else { return nil }+ return DictationInsertion(+ destination: destination, revision: self.composerRevision,+ startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))+ }}func cancelVoiceInput() {- voiceTranscriber.cancel()+ voiceTranscriber.releaseComposer()emptyComposerId = UUID()voiceInsertionCaret = nil}@@ -715,7 +721,7 @@ final class ChatViewModel: ObservableObject {}func sendDraft() {- guard !voiceTranscriber.active else { return }+ guard !voiceTranscriber.blocksComposer else { return }guard !isGenerating && !isDownloading && !isSendPending else { return }guard !isChatUnsupported else {showUnsupportedDeviceDialog = true@@ -734,6 +740,7 @@ final class ChatViewModel: ObservableObject {generationErrorMessage = ModelImagesUnsupported().localizedDescriptionreturn}+ voiceTranscriber.releaseComposer()let sourceSnapshot = currentSourceSelection()let sendId = UUID()pendingSendId = sendId@@ -1143,6 +1150,7 @@ final class ChatViewModel: ObservableObject {userNode = messageStore[sessionId]?.first(where: { $0.id == parentId })}guard let userNode else { return }+ voiceTranscriber.interrupt()let sourceSnapshot = currentSourceSelection()cancelPendingSend()priorGeneration?.cancel()@@ -1595,9 +1603,6 @@ final class ChatViewModel: ObservableObject {sessionId: parent.sessionId, promptText: prompt.text,promptImageCount: prompt.imageFiles.count, currentMessageId: parent.id,limits: limits, systemPrompt: systemPrompt)- if normalHistory.currentInputTooLarge {- throw ConversationError.CurrentInputTooLarge- }if normalHistory.wasTrimmed && overflowBypassMessageId != parent.id {return .overflow(inputTokens: normalHistory.inputTokens,@@ -2503,7 +2508,6 @@ final class ChatViewModel: ObservableObject {let inputTokens: Intlet inputBudget: Intlet wasTrimmed: Bool- let currentInputTooLarge: Bool}private struct GenerationLimits {@@ -2572,8 +2576,7 @@ final class ChatViewModel: ObservableObject {if remaining <= 0 || historyMessages.isEmpty {return HistorySelection(messages: [], inputTokens: inputTokens, inputBudget: inputBudget,- wasTrimmed: inputTokens > inputBudget,- currentInputTooLarge: systemTokens + promptTokens > inputBudget)+ wasTrimmed: inputTokens > inputBudget)}let quantum = max(1, inputBudget / 4)@@ -2603,7 +2606,7 @@ final class ChatViewModel: ObservableObject {return HistorySelection(messages: selected, inputTokens: inputTokens, inputBudget: inputBudget,- wasTrimmed: inputTokens > inputBudget, currentInputTooLarge: false)+ wasTrimmed: inputTokens > inputBudget)}private func resolveMaxOutputTokens(contextLength: Int) -> Int {diff --git a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift b/apple/apps/ensu/Ensu/Chat/MessageInputView.swiftindex f9bf3bd0d8..863aa72427 100644--- a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift+++ b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift@@ -41,7 +41,7 @@ struct MessageInputView: View {let hasContent =!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !attachments.isEmptyreturn hasContent && !isDownloading && !isSendPending- && !voiceService.active+ && !voiceService.blocksComposer}private var isImageAttachmentLimitReached: Bool {@@ -102,11 +102,12 @@ struct MessageInputView: View {.padding(.horizontal, EnsuSpacing.pageHorizontal)HStack(alignment: .bottom, spacing: EnsuSpacing.xs) {- if supportsImageAttachments && !voiceService.active {+ if supportsImageAttachments && !voiceService.blocksComposer {ActionButton(icon: "ImagePlusIcon", tooltip: "Add photos",color: EnsuColor.textPrimary, size: 32) {+ voiceService.releaseComposer()dismissInput()showPhotos = true}@@ -136,7 +137,7 @@ struct MessageInputView: View {MessageTextView(text: Binding(get: { composerText },- set: { if !voiceService.active { text = $0 } }+ set: { if !voiceService.blocksComposer { text = $0 } }),isFocused: $isFocused,moveCursorToEndToken: moveCursorToEndToken,@@ -151,8 +152,8 @@ struct MessageInputView: View {insertionCaret: voicePreview.map { Int($0.caretUtf16) }?? voiceInsertionCaret,onSelectionChange: onComposerSelection,- readOnly: voiceService.active,- maximumLines: voiceService.active ? 8 : 5+ readOnly: voiceService.blocksComposer,+ maximumLines: voiceService.blocksComposer ? 8 : 5).padding(.vertical, 6).overlay(alignment: .topLeading) {@@ -182,6 +183,7 @@ struct MessageInputView: View {onDismissKeyboard()onVoiceInput()}+ .padding(.leading, EnsuSpacing.xs)}Button {diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swiftindex 021cfc7f05..68f67b22fa 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift@@ -7,7 +7,7 @@ struct VoiceInputControls: View {private var status: String {switch service.state {- case .downloading: return "Downloading"+ case .downloading(_, let queued): return queued ? "Download queued" : "Downloading"case .preparing: return "Preparing"case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"@@ -91,7 +91,8 @@ struct VoiceInputButton: View {private var label: String {switch service.state {- case .downloading: return "Downloading voice model"+ case .downloading(_, let queued):+ return queued ? "Voice model download queued" : "Downloading voice model"case .preparing: return "Preparing voice input"case .transcribing: return "Transcribing"case .recording: return "Stop dictation"@@ -101,12 +102,17 @@ struct VoiceInputButton: View {var body: some View {Button {- if service.state.isRecording { service.stop() } else { onOpen() }+ if service.state.isRecording {+ hapticWarning()+ service.stop()+ } else {+ onOpen()+ }} label: {ZStack {switch service.state {- case .downloading(let percent):- if let percent {+ case .downloading(let percent, let queued):+ if let percent, !queued {Circle().stroke(EnsuColor.textMuted.opacity(0.2), lineWidth: 2)Circle().trim(from: 0, to: CGFloat(min(100, max(0, percent))) / 100).stroke(@@ -125,11 +131,11 @@ struct VoiceInputButton: View {RoundedRectangle(cornerRadius: 2).fill(EnsuColor.backgroundBase).frame(width: 12, height: 12)case .idle, .error:- Image("Mic02Icon").resizable().scaledToFit().frame(width: 20, height: 20)+ Image("Mic02Icon").resizable().scaledToFit().frame(width: 16, height: 16)}}.frame(width: 28, height: 28)- .frame(width: 44, height: 44)+ .frame(width: 32, height: 32).contentShape(Rectangle())}.foregroundStyle(canPress ? EnsuColor.textPrimary : EnsuColor.textMuted)diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 961975aa3d..6526995cce 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -5,7 +5,7 @@ import Combineenum VoiceInputFailure: Equatable {case permission, microphone, noSpeech, interrupted, captureInterruptedcase modelUnavailable, memoryUnavailable, storageFull, download, fileAccess, transcription- case insertion+ case insertion, incompletevar message: String {switch self {@@ -31,6 +31,8 @@ enum VoiceInputFailure: Equatable {return "Could not access voice model files. Try again."case .transcription:return "Could not transcribe voice input. Review the available text or try again."+ case .incomplete:+ return "Some words may be missing. Review the text."case .insertion:return "The draft changed before voice input finished. Tap the microphone to try again."}@@ -62,7 +64,7 @@ enum VoiceInputState: Equatable {case idlecase preparingcase recording- case downloading(percent: Int?)+ case downloading(percent: Int?, queued: Bool)case transcribingcase error(VoiceInputFailure)@@ -87,7 +89,19 @@ final class VoiceTranscriptionService: ObservableObject {@Published private(set) var snapshot: DictationSnapshot?@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)@Published private(set) var notice: VoiceInputFailure?- @Published private(set) var state: VoiceInputState = .idle+ @Published private(set) var state: VoiceInputState = .idle {+ didSet {+ transientErrorTask?.cancel()+ transientErrorTask = nil+ if state == .error(.noSpeech) {+ transientErrorTask = Task { @MainActor [weak self] in+ do { try await Task.sleep(for: .seconds(10)) } catch { return }+ guard let self, self.state == .error(.noSpeech) else { return }+ self.state = .idle+ }+ }+ }+ }@Published private(set) var isFinishingSlow = falsevar onActivityChanged: (@MainActor () -> Void)?var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?@@ -101,8 +115,10 @@ final class VoiceTranscriptionService: ObservableObject {private var recorder: LivePcmRecorder?private var task: Task<Void, Never>?private var finishingNoticeTask: Task<Void, Never>?+ private var transientErrorTask: Task<Void, Never>?private var attemptFinished = falseprivate var insertion: DictationInsertion?+ private var makeInsertion: (@MainActor () -> DictationInsertion?)?private var generation = UUID()private var foreground = trueprivate var awaitingPermission = false@@ -155,10 +171,18 @@ final class VoiceTranscriptionService: ObservableObject {}}- func open(insertion: DictationInsertion) {+ var blocksComposer: Bool {+ guard active else { return false }+ switch state {+ case .idle, .downloading: return false+ default: return true+ }+ }++ func open(makeInsertion: @escaping @MainActor () -> DictationInsertion?) {guard !active, !hasActiveTasks else { return }generation = UUID()- self.insertion = insertion+ self.makeInsertion = makeInsertionattemptFinished = falseisFinishingSlow = falsesession = DictationSession(live: usesLiveDictation)@@ -179,6 +203,7 @@ final class VoiceTranscriptionService: ObservableObject {refresh(owner, token: generation)}} else if !awaitingPermission, !startOnForeground {+ if case .downloading = state { return }interrupt()}}@@ -221,7 +246,7 @@ final class VoiceTranscriptionService: ObservableObject {}hasActiveTasks = truenotice = nil- state = .preparing+ state = .downloading(percent: nil, queued: true)task = Task { [weak self] inguard let self else { return }defer {@@ -238,19 +263,30 @@ final class VoiceTranscriptionService: ObservableObject {}do {if !self.modelAssets.allSatisfy({ self.assetStore.isDownloaded($0) }) {- self.state = .downloading(percent: nil)try await self.assetStore.download(assets: self.modelAssets) {[weak self] progress inTask { @MainActor [weak self] inguard let self, self.generation == token,case .downloading = self.stateelse { return }- self.state = .downloading(percent: Int(progress.percentage))+ self.state = .downloading(+ percent: progress.totalBytes.map { _ in Int(progress.percentage) },+ queued: progress.queued)}}}try Task.checkCancellation()- guard self.generation == token, self.foreground else { return }+ guard self.generation == token else { return }+ guard self.foreground else {+ self.cancel()+ return+ }+ guard let insertion = self.makeInsertion?() else {+ self.cancel()+ return+ }+ self.insertion = insertion+ self.makeInsertion = nilself.state = .preparinglet recorder: LivePcmRecorderdo {@@ -345,7 +381,9 @@ final class VoiceTranscriptionService: ObservableObject {let added = hasText && insertion.map { onInsert?($0, text) == true } == truelet completedState: VoiceInputStateif added {- completedState = value.incomplete ? .error(notice ?? .interrupted) : .idle+ let failure: VoiceInputFailure =+ notice == .transcription ? .incomplete : notice ?? .interrupted+ completedState = value.incomplete ? .error(failure) : .idle} else {completedState = .error(hasText ? .insertion : notice ?? .noSpeech)}@@ -354,6 +392,7 @@ final class VoiceTranscriptionService: ObservableObject {session = nilsnapshot = nilinsertion = nil+ makeInsertion = nilonInsert = nilaudioLevels = Array(repeating: 0, count: 24)state = completedState@@ -387,6 +426,10 @@ final class VoiceTranscriptionService: ObservableObject {func interrupt(reason: VoiceInputFailure = .interrupted) {guard active, !attemptFinished else { return }+ if case .downloading = state {+ releaseComposer()+ return+ }if !hasActiveTasks, stream == nil {cancel()return@@ -400,6 +443,15 @@ final class VoiceTranscriptionService: ObservableObject {state = .transcribing}+ func releaseComposer() {+ if case .downloading = state {+ makeInsertion = nil+ onInsert = nil+ } else {+ cancel()+ }+ }+func cancel() {awaitingPermission = falsestartOnForeground = false@@ -415,6 +467,7 @@ final class VoiceTranscriptionService: ObservableObject {snapshot = nilsession = nilinsertion = nil+ makeInsertion = nilstream = nilrecorder = nilonInsert = nildiff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rsindex c44343f6ac..3f6dfc3518 100644--- a/rust/bindings/uniffi/ensu/src/assets.rs+++ b/rust/bindings/uniffi/ensu/src/assets.rs@@ -90,6 +90,7 @@ pub fn knowledge_embedding_model_asset() -> Arc<Asset> {#[derive(Debug, Clone, uniffi::Record)]pub struct AssetDownloadProgress {pub label: String,+ pub queued: bool,pub downloaded_bytes: i64,pub total_bytes: Option<i64>,pub percentage: f64,@@ -102,6 +103,7 @@ impl From<ente_assets::AssetDownloadProgress> for AssetDownloadProgress {let display = ente_ensu::model::display_progress(&value);Self {label: value.asset_progress.label,+ queued: value.queued,downloaded_bytes: i64::try_from(value.batch_downloaded_bytes).unwrap_or(i64::MAX),total_bytes: value.batch_total_bytesdiff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rsindex 04e1fb4421..41668f8093 100644--- a/rust/bindings/uniffi/ensu/src/voice.rs+++ b/rust/bindings/uniffi/ensu/src/voice.rs@@ -229,7 +229,7 @@ impl DictationStream {.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_ok(){- self.session.lock().session.worker_exited(false)?;+ self.session.lock().finish(false)?;}Ok(())}@@ -244,12 +244,21 @@ impl DictationStream {let mut notified_stop = false;let mut last_notification = Instant::now();let mut on_event = |event: LiveEvent| {+ if let LiveEvent::Deferred {+ reason,+ committed_through,+ model_ready,+ } = &event+ {+ let committed_ms = committed_through / 16;+ let audio_ms = self.session.lock().session.accepted_samples() / 16;+ ente_uniffi_log::info!(+ "Dictation deferring remaining audio: reason={reason} committed_ms={committed_ms} audio_ms={audio_ms} model_ready={model_ready}"+ );+ }let immediate = matches!(&event,- LiveEvent::Commit { .. }- | LiveEvent::ReplaceTranscript(_)- | LiveEvent::Deferred- | LiveEvent::CaptureClosed+ LiveEvent::Commit { .. } | LiveEvent::Deferred { .. } | LiveEvent::CaptureClosed);if matches!(&event, LiveEvent::Ready) {let phase = self.session.lock().session.phase();@@ -291,8 +300,14 @@ impl DictationStream {state.session.observe_drained_audio(accepted, submitted)?;}}- state.session.worker_exited(result.is_ok())?;+ state.finish(result.is_ok())?;+ let reason = state.session.stop_reason();+ let audio_ms = state.session.accepted_samples() / 16;+ let incomplete = state.session.incomplete();drop(state);+ ente_uniffi_log::info!(+ "Dictation finished: reason={reason:?} audio_ms={audio_ms} incomplete={incomplete}"+ );self.stream.finish();if !notified_stop {callback.on_capture_closed();@@ -386,7 +401,7 @@ mod tests {stream.abandon(DictationStopReason::Interruption).unwrap();assert!(owner.snapshot().worker_active);assert!(DictationStream::new(owner.clone(), 16000, 1).is_err());- owner.lock().session.worker_exited(false).unwrap();+ owner.lock().finish(false).unwrap();let value = owner.snapshot();assert_eq!(format!("{}{}", value.text, value.hypothesis),@@ -436,7 +451,7 @@ mod tests {}}#[test]- fn replacement_matches_the_composer_preview_and_discard_rejects_late_text() {+ fn completion_preserves_committed_text_and_discard_rejects_late_text() {let owner = DictationSession::new(true);let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();event(@@ -463,13 +478,24 @@ mod tests {);event(&owner,- LiveEvent::ReplaceTranscript("Complete recording".into()),+ LiveEvent::Commit {+ start: 16000,+ end: 32000,+ text: "corrected tail".into(),+ },);- assert_eq!(owner.snapshot().text, "Complete recording");+ assert_eq!(owner.snapshot().text, "partial corrected tail");assert!(owner.snapshot().hypothesis.is_empty());stream.abandon(DictationStopReason::Discard).unwrap();- event(&owner, LiveEvent::ReplaceTranscript("late".into()));- assert_eq!(owner.snapshot().text, "Complete recording");+ event(+ &owner,+ LiveEvent::Commit {+ start: 32000,+ end: 48000,+ text: "late".into(),+ },+ );+ assert_eq!(owner.snapshot().text, "partial corrected tail");assert!(matches!(owner.snapshot().phase, DictationPhase::Closed));}}diff --git a/rust/bindings/uniffi/log/src/lib.rs b/rust/bindings/uniffi/log/src/lib.rsindex f7a521192c..a500dc6e5f 100644--- a/rust/bindings/uniffi/log/src/lib.rs+++ b/rust/bindings/uniffi/log/src/lib.rs@@ -1,4 +1,4 @@-pub use log::warn;+pub use log::{info, warn};#[macro_export]macro_rules! setup {diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.jsonindex 9280db1850..c9add6495a 100644--- a/rust/checks/lint-exceptions/suppressions.json+++ b/rust/checks/lint-exceptions/suppressions.json@@ -27,6 +27,7 @@"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],"rust/crates/ensu/src/platform.rs": ["unsafe_code"],+ "rust/crates/ensu/src/transcription/text.rs": ["clippy::expect_used"],"rust/crates/exif/src/isobmff.rs": ["clippy::expect_used"],"rust/crates/image/src/color_management.rs": ["clippy::expect_used"],"rust/crates/location/src/binary.rs": ["clippy::expect_used"],diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rsindex f7586beca6..9b571628da 100644--- a/rust/crates/assets/src/download.rs+++ b/rust/crates/assets/src/download.rs@@ -86,7 +86,7 @@ pub struct DownloadTarget {pub destination: PathBuf,}-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Default)]pub struct Progress {pub label: String,pub downloaded_bytes: u64,diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsindex bffde907e4..da39008c39 100644--- a/rust/crates/assets/src/store.rs+++ b/rust/crates/assets/src/store.rs@@ -42,6 +42,7 @@ pub struct AssetFile {pub struct AssetDownloadProgress {pub asset_index: usize,pub asset_progress: Progress,+ pub queued: bool,pub batch_downloaded_bytes: u64,pub batch_total_bytes: Option<u64>,pub batch_percentage: f64,@@ -218,20 +219,32 @@ impl AssetStore {.map(|asset| self.asset_lock(asset)).collect::<Result<Vec<_>, _>>()?;let progress = std::sync::Mutex::new(BatchProgress::new(assets.len(), on_progress));- let jobs = assets- .iter()- .zip(locks)- .enumerate()- .map(|(asset_index, (asset, lock))| {- self.download_asset(- asset_index,- asset,- lock,- &progress,- cancellation.clone(),- &import,- )- });+ let progress = &progress;+ let import = &import;+ let cancellation = &cancellation;+ let jobs =+ assets+ .iter()+ .zip(locks)+ .enumerate()+ .map(|(asset_index, (asset, lock))| async move {+ let result = self+ .download_asset(+ asset_index,+ asset,+ lock,+ progress,+ cancellation.clone(),+ import,+ )+ .await;+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]+ progress+ .lock()+ .expect("progress lock")+ .set_state(asset_index, AssetDownloadState::Complete);+ result+ });let results = join_all(jobs).await;if cancellation.is_cancelled() {return Err(Error::Cancelled);@@ -267,16 +280,16 @@ impl AssetStore {F: FnMut(AssetDownloadProgress) + Send,I: Fn(&Asset, &Path) -> Result<(), Error> + Sync,{+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]+ progress+ .lock()+ .expect("progress lock")+ .set_state(asset_index, AssetDownloadState::Queued);let _guard = tokio::select! {guard = lock.lock() => guard,_ = cancellation.cancelled() => return Err(Error::Cancelled),};if self.is_downloaded(asset) {- #[expect(- clippy::expect_used,- reason = "A panic unwinds the download; its local state is not reused"- )]- progress.lock().expect("progress lock").skip(asset_index);return Ok(());}import(asset, &self.staging_dir(asset))?;@@ -288,6 +301,11 @@ impl AssetStore {slot = self.coordination.asset_slots.acquire() => slot.expect("asset semaphore closed"),_ = cancellation.cancelled() => return Err(Error::Cancelled),};+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]+ progress+ .lock()+ .expect("progress lock")+ .set_state(asset_index, AssetDownloadState::Downloading);self.downloader()?.download(@@ -481,9 +499,16 @@ fn normalized_root(root: PathBuf) -> PathBuf {}}+#[derive(Clone, Copy, PartialEq, Eq)]+enum AssetDownloadState {+ Queued,+ Downloading,+ Complete,+}+struct BatchProgress<F> {assets: Vec<Option<Progress>>,- pending: Vec<bool>,+ states: Vec<AssetDownloadState>,callback: F,}@@ -494,37 +519,48 @@ wherefn new(asset_count: usize, callback: F) -> Self {Self {assets: vec![None; asset_count],- pending: vec![true; asset_count],+ states: vec![AssetDownloadState::Queued; asset_count],callback,}}- fn skip(&mut self, asset_index: usize) {- self.pending[asset_index] = false;+ fn set_state(&mut self, asset_index: usize, state: AssetDownloadState) {+ self.states[asset_index] = state;+ self.emit(+ asset_index,+ self.assets[asset_index].clone().unwrap_or_default(),+ );}fn update(&mut self, asset_index: usize, asset_progress: Progress) {- self.pending[asset_index] = false;self.assets[asset_index] = Some(asset_progress.clone());self.emit(asset_index, asset_progress);}- fn emit(&mut self, asset_index: usize, asset_progress: Progress) {+ fn emit(&mut self, asset_index: usize, mut asset_progress: Progress) {+ let queued = self.states.contains(&AssetDownloadState::Queued)+ && !self.states.contains(&AssetDownloadState::Downloading);+ if queued {+ asset_progress.label = "Download queued".to_string();+ } else if asset_progress.label.is_empty() {+ asset_progress.label = "Preparing downloads".to_string();+ }let batch_downloaded_bytes = self.assets.iter().flatten().map(|progress| progress.downloaded_bytes).sum();- let batch_total_bytes = (!self.pending.iter().any(|pending| *pending))- .then(|| {- self.assets- .iter()- .flatten()- .map(|progress| progress.total_bytes)- .try_fold(0u64, |total, value| total.checked_add(value?))+ let batch_total_bytes = self+ .assets+ .iter()+ .zip(&self.states)+ .map(|(progress, state)| match progress {+ Some(progress) => progress.total_bytes,+ None if *state == AssetDownloadState::Complete => Some(0),+ None => None,})- .flatten();+ .try_fold(0u64, |total, value| total.checked_add(value?));let batch_percentage = batch_total_bytes.filter(|total| *total > 0).map(|total| ((batch_downloaded_bytes as f64 / total as f64) * 100.0).clamp(0.0, 100.0))@@ -532,6 +568,7 @@ where(self.callback)(AssetDownloadProgress {asset_index,asset_progress,+ queued,batch_downloaded_bytes,batch_total_bytes,batch_percentage,@@ -890,24 +927,55 @@ mod tests {Asset::files(key(&["models", "second"]), vec![file("model", b"second")]).unwrap();stage_files(&store, &first, &[("model", b"first")]);stage_files(&store, &second, &[("model", b"second")]);- let mut latest_progress = None;- let mut asset_progress = [0, 0];-- store- .download(- &[first.clone(), second.clone()],- |progress| {- asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;- latest_progress = Some((- progress.batch_downloaded_bytes,- progress.batch_total_bytes,- progress.batch_percentage,- ));- },- CancellationToken::default(),- )+ let slots = store+ .coordination+ .asset_slots+ .acquire_many(ASSET_CONCURRENCY as u32).await.unwrap();+ let second_lock = store.asset_lock(&second).unwrap();+ let second_guard = second_lock.lock().await;+ let (updates, mut receiver) = tokio::sync::mpsc::unbounded_channel();+ let assets = [first.clone(), second.clone()];+ let download = store.download(+ &assets,+ |progress| updates.send(progress).unwrap(),+ CancellationToken::default(),+ );+ let mut asset_progress = [0, 0];+ let observe_queue = async {+ assert!(receiver.recv().await.unwrap().queued);+ drop(slots);+ let mut started = false;+ loop {+ let progress = receiver.recv().await.unwrap();+ asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;+ if !progress.queued {+ started = true;+ } else if progress.batch_downloaded_bytes == 5 {+ assert!(started);+ assert_eq!(progress.asset_progress.label, "Download queued");+ break;+ }+ }+ drop(second_guard);+ };+ let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(5), async {+ tokio::join!(download, observe_queue)+ })+ .await+ .unwrap();+ result.unwrap();+ let mut latest_progress = None;+ while let Ok(progress) = receiver.try_recv() {+ assert!(!progress.queued);+ asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;+ latest_progress = Some((+ progress.batch_downloaded_bytes,+ progress.batch_total_bytes,+ progress.batch_percentage,+ ));+ }assert!(store.is_downloaded(&first));assert!(store.is_downloaded(&second));diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex 34047a748e..0b523eb5ec 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -374,34 +374,21 @@ mod tests {}}- #[test]- fn live_voice_requires_more_than_four_gib() {- let four_gib = 4 * 1024 * 1024 * 1024;- for (memory, expected) in [- (None, false),- (Some(0), false),- (Some(four_gib - 1), false),- (Some(four_gib), false),- (Some(four_gib + 1), true),- (Some(8 * 1024 * 1024 * 1024), true),- ] {- assert_eq!(supports_live_voice(memory), expected, "RAM: {memory:?}");- }- }-#[test]fn mobile_ram_ladder_enforces_inclusive_boundaries_and_saved_preferences() {let gib = 1024 * 1024 * 1024;for surface in [ModelRuntimeSurface::Android, ModelRuntimeSurface::Ios] {- for (memory, expected) in [- (None, "lfm-350m"),- (Some(0), "lfm-350m"),- (Some(4 * gib), "lfm-350m"),- (Some(4 * gib + 1), "lfm-vl-1.6b"),- (Some(8 * gib), "lfm-vl-1.6b"),- (Some(8 * gib + 1), "gemma-4-e2b-q4km"),+ for (memory, expected, live_voice) in [+ (None, "lfm-350m", false),+ (Some(0), "lfm-350m", false),+ (Some(4 * gib - 1), "lfm-350m", false),+ (Some(4 * gib), "lfm-350m", false),+ (Some(4 * gib + 1), "lfm-vl-1.6b", true),+ (Some(8 * gib), "lfm-vl-1.6b", true),+ (Some(8 * gib + 1), "gemma-4-e2b-q4km", true),] {let policy = resolve_model_policy(surface, memory);+ assert_eq!(supports_live_voice(memory), live_voice, "RAM: {memory:?}");assert_eq!(policy.default_model.id, expected, "RAM: {memory:?}");assert_eq!(policy.visible_models.len(), 1);assert_eq!(policy.visible_models[0].id, expected);diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rsindex 1bba6a7a0d..9eb6e536af 100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@ -167,7 +167,9 @@ pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {pub fn display_progress(progress: &AssetDownloadProgress) -> ModelDownloadDisplay {let total = progress.batch_total_bytes.filter(|total| *total > 0);- let status = if let Some(total) = total {+ let status = if progress.queued {+ "Download queued".to_string()+ } else if let Some(total) = total {format!("Downloading... {} / {}",format_bytes(progress.batch_downloaded_bytes),diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rsindex faa79cf67c..059da6020d 100644--- a/rust/crates/ensu/src/transcription/audio.rs+++ b/rust/crates/ensu/src/transcription/audio.rs@@ -17,34 +17,35 @@ pub fn extract_speech_from_pcm16(vad_model_path: impl AsRef<Path>,input_sample_rate: u32,pcm_le: &[u8],- is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {if input_sample_rate == 0 {return Err(TranscriptionError::InvalidAudio("input sample rate must be greater than zero".to_string(),));}+ let samples = pcm16_samples(pcm_le)?;+ extract_speech(vad_model_path, input_sample_rate, &samples)+}++pub(super) fn pcm16_samples(pcm_le: &[u8]) -> Result<Vec<f32>> {if !pcm_le.len().is_multiple_of(2) {return Err(TranscriptionError::InvalidAudio("PCM16 input must contain an even number of bytes".to_string(),));}- let samples = pcm_le+ Ok(pcm_le.as_chunks::<2>().0.iter().map(|chunk| i16::from_le_bytes(*chunk) as f32 / 32768.0)- .collect::<Vec<_>>();-- extract_speech(vad_model_path, input_sample_rate, &samples, is_cancelled)+ .collect())}fn extract_speech(vad_model_path: impl AsRef<Path>,input_sample_rate: u32,samples: &[f32],- is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {if samples.is_empty() {return Ok(Vec::new());@@ -54,7 +55,7 @@ fn extract_speech(if !vad_model_path.is_file() {return Err(TranscriptionError::VadNotDownloaded);}- let silero = SileroV4Vad::new(vad_model_path)?;+ let silero = SileroV4Vad::new(vad_model_path, 0.5)?;let mut vad = SmoothedVad::new(Box::new(silero),VAD_PREFILL_FRAMES,@@ -62,14 +63,13 @@ fn extract_speech(VAD_ONSET_FRAMES,);- extract_speech_with_vad(input_sample_rate, samples, &mut vad, is_cancelled)+ extract_speech_with_vad(input_sample_rate, samples, &mut vad)}fn extract_speech_with_vad(input_sample_rate: u32,samples: &[f32],vad: &mut dyn Vad,- is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {let mut resampler = FrameResampler::new(input_sample_rate as usize,@@ -91,15 +91,7 @@ fn extract_speech_with_vad(Err(error) => vad_error = Some(error),}};- for chunk in samples.chunks(input_sample_rate as usize) {- if is_cancelled() {- return Ok(Vec::new());- }- resampler.push(chunk, &mut emit)?;- }- if is_cancelled() {- return Ok(Vec::new());- }+ resampler.push(samples, &mut emit)?;resampler.finish(&mut emit)?;if let Some(error) = vad_error {return Err(error.into());@@ -231,10 +223,9 @@ mod tests {use super::*;#[test]- fn vad_failure_and_cancellation_do_not_return_partial_recordings() {+ fn vad_failure_does_not_return_partial_recording() {struct FailsAfterSpeech {frames: usize,- fail: bool,}impl Vad for FailsAfterSpeech {fn frame_size(&self) -> usize {@@ -245,7 +236,7 @@ mod tests {_: &[f32],) -> std::result::Result<bool, transcribe_rs::TranscribeError> {self.frames += 1;- if self.frames == 1 || !self.fail {+ if self.frames == 1 {Ok(true)} else {Err(transcribe_rs::TranscribeError::Inference(@@ -255,27 +246,11 @@ mod tests {}fn reset(&mut self) {}}- let mut vad = FailsAfterSpeech {- frames: 0,- fail: true,- };+ let mut vad = FailsAfterSpeech { frames: 0 };assert!(- matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad, &mut || false), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure"))+ matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure")));assert_eq!(vad.frames, 2);- let mut vad = FailsAfterSpeech {- frames: 0,- fail: false,- };- let mut checks = 0;- let speech = extract_speech_with_vad(16000, &vec![0.2; 32000], &mut vad, &mut || {- checks += 1;- checks > 1- })- .unwrap();- assert!(speech.is_empty());- assert_eq!(checks, 2);- assert_eq!(vad.frames, 16000 / vad.frame_size());}fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {@@ -295,6 +270,13 @@ mod tests {#[test]fn native_rate_preserves_samples_and_pads_the_last_frame() {+ let pcm = [-32768_i16, -512, -1, 0, 1, 512, 32767];+ let bytes: Vec<_> = pcm.iter().flat_map(|sample| sample.to_le_bytes()).collect();+ assert_eq!(+ pcm16_samples(&bytes).unwrap(),+ pcm.map(|sample| f32::from(sample) / 32768.0)+ );+ assert!(pcm16_samples(&bytes[..bytes.len() - 1]).is_err());let input: Vec<_> = (0..961).map(|i| i as f32 / 1000.0 - 0.5).collect();let output = resample(&input, 16_000, 137);diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rsindex 9d838e9933..d821a6cbab 100644--- a/rust/crates/ensu/src/transcription/live.rs+++ b/rust/crates/ensu/src/transcription/live.rs@@ -7,16 +7,27 @@ use transcribe_rs::vad::Vad;use super::{StreamNormalizer, Transcriber};use crate::voice::{AlignedWord, CaptureFeedResult, CaptureFormat, CaptureQueue, Result, RollingRecognizer,- VoiceError,+ TranscriptBoundary, VoiceError,};pub enum LiveEvent {Ready,- Audio { start: u64, end: u64, voiced: bool },+ Audio {+ start: u64,+ end: u64,+ voiced: bool,+ },Hypothesis(String),- Commit { start: u64, end: u64, text: String },- Deferred,- ReplaceTranscript(String),+ Commit {+ start: u64,+ end: u64,+ text: String,+ },+ Deferred {+ reason: VoiceError,+ committed_through: u64,+ model_ready: bool,+ },Coverage(u64),Idle,CaptureClosed,@@ -121,8 +132,8 @@ impl LiveTranscription {.load_model().map_err(|error| super::voice_error("Load transcription model", error))},- |request| transcriber.transcribe_window(&request.samples),- |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),+ |request| transcriber.transcribe_window(&request.samples, || self.queue.is_cancelled()),+ |pcm| transcriber.transcribe_tail(pcm, || self.queue.is_cancelled()),on_event,)}@@ -132,7 +143,7 @@ impl LiveTranscription {make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,prepare: impl FnOnce() -> Result<()> + Send,mut decode: impl FnMut(&crate::voice::DecodeWindow) -> Result<Vec<AlignedWord>> + Send,- decode_recorded: impl FnOnce(&[u8]) -> Result<String>,+ decode_recorded: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,) -> Result<()> {let _capture = self.start()?;@@ -144,6 +155,8 @@ impl LiveTranscription {}let mut normalizer = StreamNormalizer::new(self.queue.format())?;let mut rolling = RollingRecognizer::default();+ let mut boundary = TranscriptBoundary::default();+ let mut provisional = Vec::new();let mut pcm = super::recorded::RecordedPcm::default();let mut accepted = 0_u64;let mut deferred = false;@@ -194,13 +207,20 @@ impl LiveTranscription {},on_event,)?;- pcm.push(&frame[..real])?;+ pcm.push(&frame[..real], voiced)?;if !deferred {match rolling.feed(&frame[..real], voiced) {Ok(()) => {}Err(VoiceError::BacklogFull) => {deferred = true;- self.emit(LiveEvent::Deferred, on_event)?;+ self.emit(+ LiveEvent::Deferred {+ reason: VoiceError::BacklogFull,+ committed_through: boundary.end,+ model_ready: prepared,+ },+ on_event,+ )?;}Err(error) => return Err(error),}@@ -244,16 +264,25 @@ impl LiveTranscription {on_event,)?;self.emit(- LiveEvent::Coverage(update.covered_through),+ LiveEvent::Coverage(update.boundary.end),on_event,)?;+ boundary = update.boundary;+ provisional = update.provisional;}Err(- VoiceError::UnstableTranscript- | VoiceError::IncompleteAudio,+ reason @ (VoiceError::UnstableTranscript+ | VoiceError::IncompleteAudio),) => {deferred = true;- self.emit(LiveEvent::Deferred, on_event)?;+ self.emit(+ LiveEvent::Deferred {+ reason,+ committed_through: boundary.end,+ model_ready: prepared,+ },+ on_event,+ )?;}Err(error) => return Err(error),}@@ -304,13 +333,9 @@ impl LiveTranscription {drop(vad);drop(normalizer);self.check()?;- let text = decode_recorded(pcm.bytes())?;+ self.finish_recording(&pcm, &boundary, &provisional, decode_recorded, on_event)?;self.check()?;- if text.trim().is_empty() {- return Err(VoiceError::IncompleteAudio);- }- self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;- self.emit(LiveEvent::Coverage(accepted), on_event)?;+ self.emit(LiveEvent::Hypothesis(String::new()), on_event)?;}Ok(())})@@ -406,7 +431,8 @@ mod tests {let (release, loading) = mpsc::sync_channel(1);let mut accepted = 0;let mut deferred = false;- let mut replacement = None;+ let mut replacement = String::new();+ let mut decoded_samples = 0;let mut coverage = 0;stream.run_with(@@ -417,8 +443,14 @@ mod tests {},|_| panic!("Initialization exceeded the existing live backlog limit"),|pcm| {- assert_eq!(pcm, [0, 32].repeat(12 * 16000));- Ok("All opening speech".into())+ assert!(pcm.len() <= super::super::MAX_RECORDED_WINDOW_SAMPLES * 2);+ assert!(pcm.as_chunks::<2>().0.iter().all(|frame| *frame == [0, 32]));+ let first = decoded_samples == 0;+ decoded_samples += pcm.len() / 2;+ super::super::recorded::decoded_chunk(+ pcm,+ if first { "All opening" } else { " speech" },+ )},&mut |event| {match event {@@ -434,8 +466,8 @@ mod tests {release.send(()).unwrap();}}- LiveEvent::Deferred => deferred = true,- LiveEvent::ReplaceTranscript(text) => replacement = Some(text),+ LiveEvent::Deferred { .. } => deferred = true,+ LiveEvent::Commit { text, .. } => replacement.push_str(&text),LiveEvent::Coverage(end) => coverage = end,_ => {}}@@ -443,7 +475,8 @@ mod tests {},).unwrap();- assert_eq!(replacement.as_deref(), Some("All opening speech"));+ assert_eq!(replacement, "All opening speech");+ assert!(decoded_samples >= 12 * 16000);assert_eq!(coverage, 12 * 16000);}@@ -548,7 +581,7 @@ mod tests {}#[test]- fn failed_live_decode_continues_capture_and_replays_every_sample_once() {+ fn failed_initial_decode_recovers_all_audio_without_ending_capture() {for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {let stream = stream();for _ in 0..3 {@@ -565,11 +598,11 @@ mod tests {|_| Err(failure.clone()),|pcm| {assert_eq!(pcm, [0, 32].repeat(9600 + 1001));- Ok("Complete recording".into())+ super::super::recorded::decoded_chunk(pcm, "Complete recording")},&mut |event| {match event {- LiveEvent::Deferred => {+ LiveEvent::Deferred { .. } => {assert!(!deferred);assert!(!closed);deferred = true;@@ -580,7 +613,7 @@ mod tests {assert!(deferred);closed = true;}- LiveEvent::ReplaceTranscript(text) => {+ LiveEvent::Commit { text, .. } => {assert!(closed);final_text.push(text);}@@ -596,29 +629,176 @@ mod tests {}}+ #[test]+ fn recovery_preserves_committed_prefix_and_cancel_keeps_the_visible_tail() {+ use super::super::DictationState;++ fn word(text: &str, start: u64, end: u64) -> AlignedWord {+ AlignedWord {+ text: text.into(),+ start,+ end,+ }+ }++ for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {+ for outcome in 0..3 {+ let stream = stream();+ let (release, loading) = mpsc::sync_channel(1);+ let mut state = DictationState::new(true);+ state.session.begin_capture().unwrap();+ let mut samples = [0, 32].repeat(192000);+ samples[128000 * 2..128512 * 2].fill(0);+ let mut fed = 0;+ let mut calls = 0;+ let mut recovered = 0;+ let mut committed = false;+ let mut deferred = false;+ let expected_failure = failure.clone();+ let recovery_failure =+ VoiceError::ProcessingFailed("injected recovery failure".into());+ let result = stream.run_with(+ vad,+ move || {+ loading.recv_timeout(Duration::from_secs(3)).unwrap();+ Ok(())+ },+ |_| {+ calls += 1;+ if calls > 3 {+ return Err(failure.clone());+ }+ let mut words = vec![+ word("Stable", 4000, 8000),+ word("tentative", 64000, 68000),+ word("tail", 72000, 76000),+ ];+ if calls == 3 {+ words.push(word("later", 144000, 152000));+ }+ Ok(words)+ },+ |pcm| {+ recovered += 1;+ if recovered == 1 {+ assert_eq!(pcm, &samples[..128512 * 2]);+ if outcome == 1 {+ stream.cancel();+ }+ Ok(vec![+ word("Stable", 4000, 8000),+ word("corrected", 64000, 68000),+ word("tail", 72000, 76000),+ ])+ } else if outcome == 2 {+ Err(recovery_failure.clone())+ } else {+ assert_eq!(recovered, 2);+ assert_eq!(pcm, &samples[52000 * 2..]);+ Ok(vec![+ word("tail", 20000, 24000),+ word("later", 92000, 100000),+ word("ending", 138000, 140000),+ ])+ }+ },+ &mut |event| {+ let feed_until = match &event {+ LiveEvent::Idle if fed < 139264 => (fed + 3200).min(139264),+ LiveEvent::Audio { end: 139264, .. } => {+ release.send(()).unwrap();+ fed+ }+ LiveEvent::Hypothesis(_) if !committed => fed + 8192,+ LiveEvent::Commit { text, .. } if !committed => {+ assert_eq!(text, "Stable");+ committed = true;+ fed + 8192+ }+ LiveEvent::Deferred {+ reason,+ committed_through,+ model_ready,+ } => {+ assert_eq!(reason, &expected_failure);+ assert_eq!(*committed_through, 8000);+ assert!(*model_ready);+ assert_eq!(state.text(), "Stable");+ assert_eq!(state.hypothesis(), " tentative tail later");+ deferred = true;+ 192000+ }+ _ => fed,+ };+ while fed < feed_until {+ let end = (fed + 3200).min(feed_until);+ stream.feed(&samples[fed * 2..end * 2])?;+ fed = end;+ }+ if matches!(event, LiveEvent::Deferred { .. }) {+ stream.finish();+ }+ state.apply(event, || false)?;+ Ok(LiveControl::Continue)+ },+ );+ assert!(deferred && committed);+ state.finish(result.is_ok()).unwrap();+ match outcome {+ 1 => {+ assert_eq!(result, Err(VoiceError::Cancelled));+ assert_eq!(state.text(), "Stable tentative tail later");+ }+ 2 => {+ assert_eq!(result, Err(recovery_failure));+ assert_eq!(state.text(), "Stable corrected tail later");+ }+ _ => {+ result.unwrap();+ assert_eq!(state.text(), "Stable corrected tail later ending");+ }+ }+ assert!(state.hypothesis().is_empty());+ assert_eq!(state.session.incomplete(), outcome != 0);+ }+ }+ }+#[test]fn incomplete_or_cancelled_recovery_cannot_publish_text_or_coverage() {- for cancelled in [false, true] {+ for (samples, cancelled) in [(1001, false), (1001, true), (32000, false), (32000, true)] {let stream = stream();- stream.feed(&[0, 32].repeat(1001)).unwrap();+ for chunk in [0, 32].repeat(samples).chunks(3200 * 2) {+ stream.feed(chunk).unwrap();+ }stream.finish();let result = stream.run_with(vad,|| Ok(()),- |_| Err(VoiceError::UnstableTranscript),|_| {+ if samples == 1001 {+ Err(VoiceError::UnstableTranscript)+ } else {+ Ok(vec![AlignedWord {+ text: "truncated prefix".into(),+ start: 0,+ end: 8000,+ }])+ }+ },+ |pcm| {if cancelled {stream.cancel();- Ok("late result".into())+ super::super::recorded::decoded_chunk(pcm, "late result")} else {- Ok(String::new())+ super::super::recorded::decoded_chunk(pcm, "")}},&mut |event| {- assert!(!matches!(- event,- LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)- ));+ assert!(!matches!(event, LiveEvent::Commit { .. }));+ if let LiveEvent::Coverage(end) = event {+ assert_eq!(end, 0);+ }Ok(LiveControl::Continue)},);diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rsindex 37720cdb56..8be3b041e3 100644--- a/rust/crates/ensu/src/transcription/mod.rs+++ b/rust/crates/ensu/src/transcription/mod.rs@@ -4,9 +4,14 @@ mod live;mod normalize;mod recorded;mod session;+mod text;mod transcriber;mod vad;+pub(super) const ENDPOINT_PADDING_SAMPLES: usize = crate::voice::VOICE_SAMPLE_RATE as usize / 2;++pub(super) const MAX_RECORDED_WINDOW_SAMPLES: usize = crate::voice::MAX_WINDOW_SAMPLES * 2;+pub use live::{LiveControl, LiveEvent, LiveTranscription};pub(crate) use normalize::{StreamNormalizer, VAD_FRAME_SAMPLES};pub use session::DictationState;diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rsindex 133eeaf934..d076e2f51a 100644--- a/rust/crates/ensu/src/transcription/recorded.rs+++ b/rust/crates/ensu/src/transcription/recorded.rs@@ -1,75 +1,192 @@+use std::ops::Range;use std::time::Duration;use transcribe_rs::vad::Vad;use super::{LiveControl, LiveEvent, LiveTranscription, StreamNormalizer, Transcriber};-use crate::voice::{Result, VoiceError};+use crate::voice::{AlignedWord, RecognitionUpdate, Result, TranscriptBoundary, VoiceError};-const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;+const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 4) * 16_000;-pub(super) fn transcribe_chunks(- mut samples: &[f32],- mut decode: impl FnMut(&[f32]) -> super::Result<Option<String>>,-) -> super::Result<Option<String>> {- use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE, transcript_separator};+pub(super) fn chunk_end(samples: &[f32]) -> usize {+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};- let mut text = String::new();- while !samples.is_empty() {- let end = if samples.len() <= MAX_WINDOW_SAMPLES {- samples.len()- } else {- let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);- let start = limit - 2 * VOICE_SAMPLE_RATE as usize;- let offset = samples[start..limit]- .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()- .0- .iter()- .enumerate()- .map(|(index, frame)| {- (- index,- frame.iter().map(|sample| sample * sample).sum::<f32>(),- )- })- .min_by(|a, b| a.1.total_cmp(&b.1))- .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);- start + offset- };- let Some(chunk) = decode(&samples[..end])? else {- return Ok(None);- };- let chunk = chunk.trim();- text.push_str(transcript_separator(&text, chunk));- text.push_str(chunk);- samples = &samples[end..];+ if samples.len() <= MAX_WINDOW_SAMPLES {+ samples.len()+ } else {+ let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);+ let start = limit - 2 * VOICE_SAMPLE_RATE as usize;+ let offset = samples[start..limit]+ .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()+ .0+ .iter()+ .enumerate()+ .map(|(index, frame)| {+ (+ index,+ frame.iter().map(|sample| sample * sample).sum::<f32>(),+ )+ })+ .min_by(|a, b| a.1.total_cmp(&b.1))+ .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);+ start + offset}- Ok(Some(text))}#[derive(Default)]-pub(super) struct RecordedPcm(Vec<u8>);+pub(super) struct RecordedPcm {+ bytes: Vec<u8>,+ speech: Vec<Range<usize>>,+}impl RecordedPcm {- pub fn push(&mut self, samples: &[f32]) -> Result<()> {- if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(self.0.len() / 2) {+ pub fn push(&mut self, samples: &[f32], voiced: bool) -> Result<()> {+ let start = self.bytes.len() / 2;+ if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(start) {return Err(VoiceError::BacklogFull);}- self.0.extend(samples.iter().flat_map(|sample| {+ self.bytes.extend(samples.iter().flat_map(|sample| {((sample * 32768.0).round().clamp(-32768.0, 32767.0) as i16).to_le_bytes()}));+ if voiced && !samples.is_empty() {+ let end = start + samples.len();+ if let Some(previous) = self.speech.last_mut().filter(|range| range.end == start) {+ previous.end = end;+ } else {+ self.speech.push(start..end);+ }+ }Ok(())}+ #[cfg(test)]pub fn bytes(&self) -> &[u8] {- &self.0+ &self.bytes+ }++ pub fn transcribe(+ &self,+ boundary: &TranscriptBoundary,+ provisional: &[AlignedWord],+ mut decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,+ mut on_update: impl FnMut(RecognitionUpdate) -> Result<()>,+ ) -> Result<()> {+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};++ let total = self.bytes.len() / 2;+ let mut boundary = boundary.clone();+ let mut provisional = provisional.to_vec();+ if self.speech.is_empty() && provisional.is_empty() {+ boundary.end = total as u64;+ return on_update(boundary.pending_update(provisional));+ }+ while boundary.end < total as u64 {+ let start = boundary.start() as usize;+ let available = (start + MAX_WINDOW_SAMPLES + VOICE_SAMPLE_RATE as usize).min(total);+ let samples = super::audio::pcm16_samples(&self.bytes[start * 2..available * 2])+ .map_err(|error| super::voice_error("Read recorded audio", error))?;+ let normal_end = start + chunk_end(&samples);+ let recovery_end = (start + super::MAX_RECORDED_WINDOW_SAMPLES).min(total);+ let mut progressed = false;+ let mut failure = VoiceError::IncompleteAudio;+ for end in [normal_end, recovery_end] {+ if end as u64 <= boundary.end {+ continue;+ }+ let result = decode(&self.bytes[start * 2..end * 2]).and_then(|words| {+ let update =+ boundary.recorded_update(words, &provisional, end - start, end == total)?;+ if update.commit.is_none()+ && update.provisional.is_empty()+ && !self.has_speech(boundary.end as usize..end)+ {+ let mut next = TranscriptBoundary::default();+ next.end = if end == total {+ end as u64+ } else {+ (end as u64 - VOICE_SAMPLE_RATE * 3 / 2).max(boundary.end)+ };+ return Ok(next.pending_update(Vec::new()));+ }+ Ok(update)+ });+ match result {+ Ok(update) => {+ provisional = update.provisional.clone();+ if update.boundary.end <= boundary.end {+ on_update(update)?;+ failure = VoiceError::IncompleteAudio;+ } else {+ boundary = update.boundary.clone();+ on_update(update)?;+ if end == total {+ if self.has_speech(+ (boundary.end as usize + super::ENDPOINT_PADDING_SAMPLES)+ .min(total)..total,+ ) {+ return Err(VoiceError::IncompleteAudio);+ }+ if boundary.end < total as u64 {+ boundary.end = total as u64;+ on_update(boundary.pending_update(provisional))?;+ }+ return Ok(());+ }+ progressed = true;+ break;+ }+ }+ Err(error @ VoiceError::IncompleteAudio) => {+ failure = error;+ }+ Err(error) => {+ if error != VoiceError::Cancelled {+ on_update(boundary.pending_update(provisional))?;+ }+ return Err(error);+ }+ }+ if normal_end == recovery_end {+ break;+ }+ }+ if !progressed {+ on_update(boundary.pending_update(provisional))?;+ return Err(failure);+ }+ }+ Ok(())+ }++ fn has_speech(&self, range: Range<usize>) -> bool {+ let index = self+ .speech+ .partition_point(|speech| speech.end <= range.start);+ range.start < range.end+ && self+ .speech+ .get(index)+ .is_some_and(|speech| speech.start < range.end)}}impl Drop for RecordedPcm {fn drop(&mut self) {use zeroize::Zeroize;- self.0.zeroize();+ self.bytes.zeroize();+ }+}++#[cfg(test)]+pub(super) fn decoded_chunk(pcm: &[u8], text: &str) -> Result<Vec<AlignedWord>> {+ if text.is_empty() {+ return Ok(Vec::new());}+ Ok(vec![AlignedWord {+ text: text.into(),+ start: (pcm.len() as u64 / 2).saturating_sub(1600),+ end: pcm.len() as u64 / 2,+ }])}impl LiveTranscription {@@ -80,7 +197,7 @@ impl LiveTranscription {) -> Result<()> {self.run_recorded_with(|| transcriber.live_vad(),- |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),+ |pcm| transcriber.transcribe_tail(pcm, || self.queue.is_cancelled()),on_event,)}@@ -88,7 +205,7 @@ impl LiveTranscription {fn run_recorded_with(&self,make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,- decode: impl FnOnce(&[u8]) -> Result<String>,+ decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,) -> Result<()> {let _capture = self.start()?;@@ -97,7 +214,6 @@ impl LiveTranscription {self.emit(LiveEvent::Ready, on_event)?;let mut pcm = RecordedPcm::default();let mut accepted = 0_u64;- let mut heard_speech = false;let mut notified_stop = false;loop {self.check()?;@@ -107,7 +223,6 @@ impl LiveTranscription {let voiced = vad.is_speech(frame).map_err(|error| {super::voice_error("Detect speech in recorded audio", error)})?;- heard_speech |= voiced;let start = accepted;accepted += real as u64;self.emit(@@ -118,7 +233,7 @@ impl LiveTranscription {},on_event,)?;- pcm.push(&frame[..real])+ pcm.push(&frame[..real], voiced)};if let Some(chunk) = chunk {normalizer.push(&chunk, &mut emit)?;@@ -142,15 +257,35 @@ impl LiveTranscription {drop(vad);drop(normalizer);self.check()?;- if heard_speech {- let text = decode(pcm.bytes())?;- self.check()?;- if !text.trim().is_empty() {- self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;- }- }- self.emit(LiveEvent::Coverage(accepted), on_event)?;- Ok(())+ self.finish_recording(&pcm, &TranscriptBoundary::default(), &[], decode, on_event)+ }++ pub(super) fn finish_recording(+ &self,+ pcm: &RecordedPcm,+ boundary: &TranscriptBoundary,+ provisional: &[AlignedWord],+ mut decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,+ ) -> Result<()> {+ pcm.transcribe(+ boundary,+ provisional,+ |bytes| {+ self.check()?;+ let words = decode(bytes)?;+ self.check()?;+ Ok(words)+ },+ |update| {+ self.check()?;+ if let Some((start, end, text)) = update.commit {+ self.emit(LiveEvent::Commit { start, end, text }, on_event)?;+ }+ self.emit(LiveEvent::Hypothesis(update.hypothesis), on_event)?;+ self.emit(LiveEvent::Coverage(update.boundary.end), on_event)+ },+ )}}@@ -167,41 +302,130 @@ mod tests {let mut samples = vec![0.25; MAX_WINDOW_SAMPLES * 3 + 1];let quiet = MAX_WINDOW_SAMPLES - 2 * 16000 + 10 * super::super::VAD_FRAME_SAMPLES;samples[quiet..quiet + super::super::VAD_FRAME_SAMPLES].fill(0.0);- let mut offset = 0;- let mut decoded = 0;- let text = transcribe_chunks(&samples, |chunk| {- assert!(chunk.len() <= MAX_WINDOW_SAMPLES);- assert!(chunk.len() >= 16000);- assert_eq!(chunk, &samples[offset..offset + chunk.len()]);- if offset == 0 {- assert_eq!(chunk.len(), quiet + super::super::VAD_FRAME_SAMPLES);- }- offset += chunk.len();- decoded += 1;- Ok(Some(" again ".into()))- })- .unwrap()- .unwrap();- assert_eq!(offset, samples.len());- assert_eq!(text.split_whitespace().count(), decoded);+ let mut pcm = RecordedPcm::default();+ pcm.push(&samples, true).unwrap();let mut calls = 0;- assert!(- transcribe_chunks(&samples, |_| {+ let mut previous_window_end = 0;+ let mut covered = 0;+ let boundary = std::cell::RefCell::new(TranscriptBoundary::default());+ pcm.transcribe(+ &TranscriptBoundary::default(),+ &[],+ |bytes| {+ let boundary = boundary.borrow();+ let start = boundary.start() as usize;+ let end = start + bytes.len() / 2;+ assert!(bytes.len() <= MAX_WINDOW_SAMPLES * 2);+ if calls == 0 {+ assert_eq!(end, quiet + super::super::VAD_FRAME_SAMPLES);+ } else {+ assert!(start < boundary.end as usize);+ assert!(boundary.end < previous_window_end as u64);+ }+ previous_window_end = end;calls += 1;- Ok((calls == 1).then(|| "unfinished".into()))- })- .unwrap()- .is_none()+ let word_end = if end == samples.len() {+ end+ } else {+ end - 32000+ };+ Ok(vec![AlignedWord {+ text: "again".into(),+ start: (word_end - start - 1600) as u64,+ end: (word_end - start) as u64,+ }])+ },+ |update| {+ assert!(update.boundary.end > covered);+ covered = update.boundary.end;+ *boundary.borrow_mut() = update.boundary;+ Ok(())+ },+ )+ .unwrap();+ assert_eq!(covered, samples.len() as u64);+ assert!(calls > 3);+ let mut calls = 0;+ let mut pending = String::new();+ assert_eq!(+ pcm.transcribe(+ &TranscriptBoundary::default(),+ &[],+ |bytes| {+ calls += 1;+ if calls == 1 {+ decoded_chunk(bytes, "unfinished")+ } else {+ Err(VoiceError::Cancelled)+ }+ },+ |update| {+ assert!(update.commit.is_none());+ pending = update.hypothesis;+ Ok(())+ },+ ),+ Err(VoiceError::Cancelled));assert_eq!(calls, 2);+ assert_eq!(pending, "unfinished");++ let mut pcm = RecordedPcm::default();+ pcm.push(&vec![0.0; MAX_WINDOW_SAMPLES], false).unwrap();+ pcm.push(&vec![0.25; 16000], true).unwrap();+ let mut decoded = 0;+ let mut covered = 0;+ assert_eq!(+ pcm.transcribe(+ &TranscriptBoundary::default(),+ &[],+ |pcm| {+ decoded += 1;+ decoded_chunk(pcm, "")+ },+ |update| {+ assert!(update.commit.is_none() && update.hypothesis.is_empty());+ covered = update.boundary.end;+ Ok(())+ },+ ),+ Err(VoiceError::IncompleteAudio)+ );+ assert_eq!(decoded, 2);+ assert!(covered > 0 && covered <= MAX_WINDOW_SAMPLES as u64);}#[test]- fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {- let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);- pcm.push(&[0.25]).unwrap();+ fn recorded_capacity_drains_resampler_and_rejects_overflow_atomically() {+ let prefix = (30 * 60 - 2) * 16_000;+ let mut pcm = RecordedPcm {+ bytes: vec![0; prefix * 2],+ speech: Vec::new(),+ };+ let mut normalizer = StreamNormalizer::new(CaptureFormat {+ sample_rate: 47_999,+ channels: 1,+ })+ .unwrap();+ let input = vec![0; 4799 * 2];+ let accepted = std::cell::Cell::new(0);+ let mut emit = |frame: &[f32], real: usize| {+ accepted.set(accepted.get() + real);+ pcm.push(&frame[..real], true)+ };+ while accepted.get() < 2 * 16_000 {+ normalizer.push(&input, &mut emit).unwrap();+ }+ for _ in 0..20 {+ normalizer.push(&input, &mut emit).unwrap();+ }+ normalizer.finish(&mut emit).unwrap();+ assert!(accepted.get() > 5 * 16_000);+ assert_eq!(pcm.bytes().len(), (prefix + accepted.get()) * 2);+ pcm.bytes.resize((MAX_RECORDED_SAMPLES - 1) * 2, 0);+ pcm.push(&[0.25], true).unwrap();assert_eq!(pcm.bytes().len(), MAX_RECORDED_SAMPLES * 2);- assert_eq!(pcm.push(&[0.5]), Err(VoiceError::BacklogFull));+ assert_eq!(pcm.push(&[0.5], true), Err(VoiceError::BacklogFull));assert_eq!(&pcm.bytes()[pcm.bytes().len() - 2..],&8192_i16.to_le_bytes()@@ -210,17 +434,21 @@ mod tests {#[test]fn recorded_completion_preserves_the_tail_and_cancellation_suppresses_results() {- for cancelled in [false, true] {+ use super::super::DictationState;++ for (cancelled, incomplete) in [(false, false), (true, false), (false, true)] {let stream = LiveTranscription::new(CaptureFormat {sample_rate: 16000,channels: 1,}).unwrap();- stream.feed(&[0, 32].repeat(1001)).unwrap();- stream.finish();+ let audio = [0, 32].repeat(12 * 16000 + 1001);+ let mut chunks = audio.chunks(6400);+ let mut state = DictationState::new(false);+ state.session.begin_capture().unwrap();let mut closed = 0;- let mut commits = Vec::new();- let mut covered = None;+ let mut decoded = 0;+ let mut covered = 0;let result = stream.run_recorded_with(|| {Ok(Box::new(EnergyVad::new(@@ -229,35 +457,62 @@ mod tests {)))},|pcm| {- assert_eq!(pcm, [0, 32].repeat(1001));+ assert!(+ !pcm.is_empty()+ && pcm.len() <= super::super::MAX_RECORDED_WINDOW_SAMPLES * 2+ );+ decoded += 1;+ if decoded == 1 {+ let end = pcm.len() as u64 / 2 - 32000;+ return Ok(vec![AlignedWord {+ text: "dictated words".into(),+ start: end - 1600,+ end,+ }]);+ }if cancelled {stream.cancel();}- Ok("dictated words".into())+ decoded_chunk(pcm, if incomplete { "" } else { "continued" })},&mut |event| {match event {- LiveEvent::CaptureClosed => closed += 1,- LiveEvent::ReplaceTranscript(text) => {- assert_eq!(closed, 1);- commits.push(text);+ LiveEvent::Ready | LiveEvent::Idle => {+ if let Some(chunk) = chunks.next() {+ stream.feed(chunk)?;+ } else {+ stream.finish();+ }}- LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),- LiveEvent::Coverage(end) => covered = Some(end),+ LiveEvent::CaptureClosed => closed += 1,+ LiveEvent::Commit { .. } => assert_eq!(closed, 1),+ LiveEvent::Coverage(end) => covered = end,_ => {}}+ state.apply(event, || false)?;Ok(LiveControl::Continue)},);assert_eq!(closed, 1);- if cancelled {- assert_eq!(result, Err(VoiceError::Cancelled));- assert!(commits.is_empty());- assert_eq!(covered, None);+ assert_eq!(decoded, 2);+ state.session.worker_exited(result.is_ok()).unwrap();+ if cancelled || incomplete {+ assert_eq!(+ result,+ Err(if cancelled {+ VoiceError::Cancelled+ } else {+ VoiceError::IncompleteAudio+ })+ );+ assert_eq!(state.text(), "dictated words");+ assert!(covered > 0 && covered < audio.len() as u64 / 2);+ assert!(state.session.incomplete());} else {result.unwrap();- assert_eq!(commits, ["dictated words"]);- assert_eq!(covered, Some(1001));+ assert_eq!(state.text(), "dictated words continued");+ assert_eq!(covered, audio.len() as u64 / 2);+ assert!(!state.session.incomplete());}}}@@ -289,7 +544,7 @@ mod tests {},|pcm| {assert_eq!(pcm, audio);- Ok("Complete recorded transcript".into())+ decoded_chunk(pcm, "Complete recorded transcript")},&mut |event| {let idle = matches!(event, LiveEvent::Idle);diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rsindex cee8f3ae32..c4755d36cd 100644--- a/rust/crates/ensu/src/transcription/session.rs+++ b/rust/crates/ensu/src/transcription/session.rs@@ -32,6 +32,17 @@ impl DictationState {)}+ pub fn finish(&mut self, successful: bool) -> voice::Result<()> {+ self.session.worker_exited(successful)?;+ self.text = super::text::filter_transcription_output(&format!(+ "{}{}",+ self.text,+ self.hypothesis()+ ));+ self.hypothesis.clear();+ Ok(())+ }+pub fn apply(&mut self,event: LiveEvent,@@ -70,16 +81,7 @@ impl DictationState {self.text.push_str(&text);self.hypothesis.clear();}- LiveEvent::ReplaceTranscript(text) => {- if text.trim().is_empty() {- return Err(voice::VoiceError::IncompleteAudio);- }- self.session.observe_recognition(true)?;- self.text = text;- self.hypothesis.clear();- self.last_commit_end = u64::MAX;- }- LiveEvent::Deferred => self.session.defer_recognition(),+ LiveEvent::Deferred { .. } => self.session.defer_recognition(),LiveEvent::Coverage(end) => self.session.recognize_through(end)?,LiveEvent::Idle => {if self.session.silence_elapsed() && finish_if_empty() {@@ -104,6 +106,56 @@ mod tests {state.apply(event, || true).unwrap();}+ #[test]+ fn cleanup_waits_for_completion_and_spans_commits_and_unfinished_hypotheses() {+ for successful in [false, true] {+ let mut state = DictationState::new(true);+ state.session.begin_capture().unwrap();+ event(+ &mut state,+ LiveEvent::Audio {+ start: 0,+ end: 32000,+ voiced: true,+ },+ );+ event(+ &mut state,+ LiveEvent::Commit {+ start: 0,+ end: 16000,+ text: "um I I".into(),+ },+ );+ event(&mut state, LiveEvent::Hypothesis("I uh think".into()));+ assert_eq!(state.text(), "um I I");+ assert_eq!(state.hypothesis(), " I uh think");+ if successful {+ event(+ &mut state,+ LiveEvent::Commit {+ start: 16000,+ end: 32000,+ text: "I uh think".into(),+ },+ );+ event(&mut state, LiveEvent::Coverage(32000));+ }+ state+ .session+ .stop(if successful {+ voice::StopReason::Done+ } else {+ voice::StopReason::Interruption+ })+ .unwrap();+ state.finish(successful).unwrap();+ assert_eq!(state.text(), "I think");+ assert!(state.hypothesis().is_empty());+ assert_eq!(state.session.incomplete(), !successful);+ }+ }+#[test]fn queued_audio_is_processed_before_silence_can_stop_capture() {let mut state = DictationState::new(false);@@ -158,7 +210,7 @@ mod tests {event(state, LiveEvent::Commit { start, end, text });}event(state, LiveEvent::Hypothesis(update.hypothesis));- event(state, LiveEvent::Coverage(update.covered_through));+ event(state, LiveEvent::Coverage(update.boundary.end));}for (words, expected) in [@@ -199,13 +251,14 @@ mod tests {.collect()};for i in 0..3 {- rolling.feed(&vec![0.2; 16000], true).unwrap();+ rolling.feed(&vec![0.2; 8000], i == 0).unwrap();+ rolling.feed(&vec![0.2; 8000], false).unwrap();event(&mut state,LiveEvent::Audio {start: i * 16000,end: (i + 1) * 16000,- voiced: true,+ voiced: i == 0,},);let request = rolling.next_window().unwrap().unwrap();diff --git a/rust/crates/ensu/src/transcription/text.rs b/rust/crates/ensu/src/transcription/text.rsnew file mode 100644index 0000000000..781fbbd952--- /dev/null+++ b/rust/crates/ensu/src/transcription/text.rs@@ -0,0 +1,81 @@+use regex::{Regex, regex};+use std::sync::LazyLock;++const FILLER_WORDS: &[&str] = &[+ "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha",+ "ehh",+];++#[expect(+ clippy::expect_used,+ reason = "Filler words are escaped before insertion into the fixed regex"+)]+static FILLER_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {+ FILLER_WORDS+ .iter()+ .map(|word| {+ Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word)))+ .expect("escaped filler word forms a valid regex")+ })+ .collect()+});++pub fn filter_transcription_output(text: &str) -> String {+ let mut filtered = text.to_string();++ for pattern in FILLER_PATTERNS.iter() {+ filtered = pattern.replace_all(&filtered, "").to_string();+ }++ filtered = collapse_stutters(&filtered);+ filtered = regex!(r"\s{2,}").replace_all(&filtered, " ").to_string();+ filtered.trim().to_string()+}++fn collapse_stutters(text: &str) -> String {+ let words = text.split_whitespace().collect::<Vec<_>>();+ if words.is_empty() {+ return text.to_string();+ }++ let mut result = Vec::new();+ let mut i = 0;++ while i < words.len() {+ let word = words[i];+ let word_lower = word.to_lowercase();++ if word_lower.len() <= 2 && word_lower.chars().all(char::is_alphabetic) {+ let mut count = 1;+ while i + count < words.len() && words[i + count].to_lowercase() == word_lower {+ count += 1;+ }++ result.push(word);+ i += if count >= 3 { count } else { 1 };+ } else {+ result.push(word);+ i += 1;+ }+ }++ result.join(" ")+}++#[cfg(test)]+mod tests {+ use super::filter_transcription_output;++ #[test]+ fn removes_filler_words() {+ assert_eq!(+ filter_transcription_output("um hello uh world"),+ "hello world"+ );+ }++ #[test]+ fn collapses_repeated_short_stutters() {+ assert_eq!(filter_transcription_output("I I I think"), "I think");+ }+}diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rsindex a23c23afed..392edb35c6 100644--- a/rust/crates/ensu/src/transcription/transcriber.rs+++ b/rust/crates/ensu/src/transcription/transcriber.rs@@ -3,7 +3,7 @@ use std::sync::{Mutex, MutexGuard, PoisonError};use transcribe_rs::onnx::Quantization;use transcribe_rs::onnx::parakeet::{ParakeetModel, ParakeetParams, TimestampGranularity};-use transcribe_rs::{OrtAccelerator, get_ort_accelerator};+use transcribe_rs::{OrtAccelerator, TranscriptionResult, get_ort_accelerator};use super::graph_cache::{GRAPHS, GraphCache, LoadPlan, SOURCES};use crate::transcription::audio::extract_speech_from_pcm16;@@ -28,7 +28,7 @@ impl Transcriber {if !metadata.is_file() {return Err(crate::voice::VoiceError::VadNotDownloaded);}- super::SileroV4Vad::new(&self.vad_model_path)+ super::SileroV4Vad::new(&self.vad_model_path, 0.3).map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>).map_err(|error| super::voice_error("Load voice activity model", error))}@@ -36,8 +36,12 @@ impl Transcriber {pub(crate) fn transcribe_window(&self,samples: &[f32],+ mut is_cancelled: impl FnMut() -> bool,) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {- use crate::voice::{AlignedWord, MAX_WINDOW_SAMPLES, VoiceError};+ use crate::voice::{MAX_WINDOW_SAMPLES, VoiceError};+ if is_cancelled() {+ return Err(VoiceError::Cancelled);+ }if samples.is_empty()|| samples.len() > MAX_WINDOW_SAMPLES|| samples.iter().any(|s| !s.is_finite())@@ -52,34 +56,10 @@ impl Transcriber {true,).map_err(|error| super::voice_error("Load transcription model", error))?;- let result = model- .transcribe_with(- samples,- &ParakeetParams {- timestamp_granularity: Some(TimestampGranularity::Word),- ..Default::default()- },- )- .map_err(|error| super::voice_error("Transcribe live audio", error))?;- result- .segments- .ok_or(VoiceError::UnstableTranscript)?- .into_iter()- .map(|word| {- if !word.start.is_finite()- || !word.end.is_finite()- || word.end < word.start- || word.start * 16000.0 > samples.len() as f32- {- return Err(VoiceError::UnstableTranscript);- }- Ok(AlignedWord {- text: word.text,- start: (word.start.max(0.0) * 16000.0).round() as u64,- end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples.len() as u64),- })- })- .collect()+ let result = decode_window(model, samples, &mut is_cancelled)+ .map_err(|error| super::voice_error("Transcribe live audio", error))?+ .ok_or(VoiceError::Cancelled)?;+ aligned_words(result, samples.len())}pub fn new(model_dir: impl Into<PathBuf>, vad_model_path: impl Into<PathBuf>) -> Self {@@ -120,31 +100,8 @@ impl Transcriber {}pub fn transcribe(&self, input_sample_rate: u32, pcm_le: &[u8]) -> Result<String> {- self.transcribe_with_cancel(input_sample_rate, pcm_le, || false)- .map(Option::unwrap_or_default)- }-- pub(crate) fn transcribe_recording(- &self,- pcm_le: &[u8],- is_cancelled: impl FnMut() -> bool,- ) -> crate::voice::Result<String> {- self.transcribe_with_cancel(16_000, pcm_le, is_cancelled)- .map_err(|error| super::voice_error("Transcribe recorded audio", error))?- .ok_or(crate::voice::VoiceError::Cancelled)- }-- fn transcribe_with_cancel(- &self,- input_sample_rate: u32,- pcm_le: &[u8],- mut is_cancelled: impl FnMut() -> bool,- ) -> Result<Option<String>> {- if is_cancelled() {- return Ok(None);- }if pcm_le.is_empty() {- return Ok(Some(String::new()));+ return Ok(String::new());}if !self.model_dir.is_dir() {return Err(TranscriptionError::NotDownloaded);@@ -152,40 +109,56 @@ impl Transcriber {if !self.vad_model_path.is_file() {return Err(TranscriptionError::VadNotDownloaded);}-- let speech = extract_speech_from_pcm16(- &self.vad_model_path,- input_sample_rate,- pcm_le,- &mut is_cancelled,+ let speech = extract_speech_from_pcm16(&self.vad_model_path, input_sample_rate, pcm_le)?;+ if speech.is_empty() {+ return Ok(String::new());+ }+ let mut loaded = self.lock();+ let model = ensure_loaded(+ &mut loaded,+ &self.model_dir,+ self.graph_cache.as_ref(),+ true,+ )?;+ let result = model.transcribe_with(+ &speech,+ &ParakeetParams {+ timestamp_granularity: Some(TimestampGranularity::Segment),+ ..Default::default()+ },)?;+ Ok(super::text::filter_transcription_output(&result.text))+ }++ pub(crate) fn transcribe_tail(+ &self,+ pcm_le: &[u8],+ mut is_cancelled: impl FnMut() -> bool,+ ) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {+ use crate::voice::VoiceError;+if is_cancelled() {- return Ok(None);+ return Err(VoiceError::Cancelled);}- if speech.is_empty() {- return Ok(Some(String::new()));+ let samples = super::audio::pcm16_samples(pcm_le)+ .map_err(|error| super::voice_error("Read recorded audio", error))?;+ if samples.is_empty() || samples.len() > super::MAX_RECORDED_WINDOW_SAMPLES {+ return Err(VoiceError::InvalidAudio(+ "Recorded audio exceeds the recognition window".into(),+ ));}-let mut loaded = self.lock();let model = ensure_loaded(&mut loaded,&self.model_dir,self.graph_cache.as_ref(),false,- )?;- super::recorded::transcribe_chunks(&speech, |samples| {- if is_cancelled() {- return Ok(None);- }- let result = model.transcribe_with(- samples,- &ParakeetParams {- timestamp_granularity: Some(TimestampGranularity::Segment),- ..Default::default()- },- )?;- Ok((!is_cancelled()).then_some(result.text))- })+ )+ .map_err(|error| super::voice_error("Load transcription model", error))?;+ let result = decode_window(model, &samples, &mut is_cancelled)+ .map_err(|error| super::voice_error("Transcribe recorded audio", error))?+ .ok_or(VoiceError::Cancelled)?;+ aligned_words(result, samples.len())}fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {@@ -193,6 +166,63 @@ impl Transcriber {}}+fn decode_window(+ model: &mut ParakeetModel,+ samples: &[f32],+ is_cancelled: &mut impl FnMut() -> bool,+) -> Result<Option<TranscriptionResult>> {+ if is_cancelled() {+ return Ok(None);+ }+ let params = ParakeetParams {+ timestamp_granularity: Some(TimestampGranularity::Word),+ ..Default::default()+ };+ let mut result = model.transcribe_with(samples, ¶ms)?;+ if is_cancelled() {+ return Ok(None);+ }+ let maximum = if samples.len() <= crate::voice::MAX_WINDOW_SAMPLES {+ crate::voice::MAX_WINDOW_SAMPLES+ } else {+ super::MAX_RECORDED_WINDOW_SAMPLES+ };+ let padding = (maximum - samples.len()).min(super::ENDPOINT_PADDING_SAMPLES);+ if result.text.trim().is_empty() && padding > 0 {+ let mut padded = Vec::with_capacity(samples.len() + padding);+ padded.extend_from_slice(samples);+ padded.resize(samples.len() + padding, 0.0);+ result = model.transcribe_with(&padded, ¶ms)?;+ }+ Ok((!is_cancelled()).then_some(result))+}++fn aligned_words(+ result: TranscriptionResult,+ samples: usize,+) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {+ use crate::voice::{AlignedWord, VoiceError};+ result+ .segments+ .ok_or(VoiceError::UnstableTranscript)?+ .into_iter()+ .map(|word| {+ if !word.start.is_finite()+ || !word.end.is_finite()+ || word.end < word.start+ || word.start * 16000.0 > samples as f32+ {+ return Err(VoiceError::UnstableTranscript);+ }+ Ok(AlignedWord {+ text: word.text,+ start: (word.start.max(0.0) * 16000.0).round() as u64,+ end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples as u64),+ })+ })+ .collect()+}+fn ensure_loaded<'a>(loaded: &'a mut Option<ParakeetModel>,model_dir: &Path,diff --git a/rust/crates/ensu/src/transcription/vad.rs b/rust/crates/ensu/src/transcription/vad.rsindex 8869d0139c..b1beb87037 100644--- a/rust/crates/ensu/src/transcription/vad.rs+++ b/rust/crates/ensu/src/transcription/vad.rs@@ -12,6 +12,7 @@ pub(crate) struct SileroV4Vad {session: Session,hidden: [f32; 128],cell: [f32; 128],+ threshold: f32,}fn inference(error: impl std::fmt::Display) -> TranscribeError {@@ -19,7 +20,7 @@ fn inference(error: impl std::fmt::Display) -> TranscribeError {}impl SileroV4Vad {- pub fn new(path: impl AsRef<Path>) -> Result<Self, TranscribeError> {+ pub fn new(path: impl AsRef<Path>, threshold: f32) -> Result<Self, TranscribeError> {let session = Session::builder().map_err(inference)?.with_optimization_level(GraphOptimizationLevel::Level3)@@ -34,6 +35,7 @@ impl SileroV4Vad {session,hidden: [0.0; 128],cell: [0.0; 128],+ threshold,})}@@ -97,7 +99,7 @@ impl Vad for SileroV4Vad {SILERO_V4_FRAME_SAMPLES}fn is_speech(&mut self, frame: &[f32]) -> Result<bool, TranscribeError> {- Ok(self.speech_probability(frame)? > 0.5)+ Ok(self.speech_probability(frame)? > self.threshold)}fn reset(&mut self) {self.hidden.fill(0.0);diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rsindex 2d466e5a66..3fa2d5a69b 100644--- a/rust/crates/ensu/src/voice/mod.rs+++ b/rust/crates/ensu/src/voice/mod.rs@@ -9,9 +9,8 @@ pub(crate) use capture::CaptureQueue;pub use capture::{CaptureFeedResult, CaptureFormat};pub(crate) use composer::transcript_separator;pub use composer::{ComposerAnchor, ComposerTransfer, VoiceInsertion, preview_insertion};-#[cfg(test)]pub(crate) use rolling::RecognitionUpdate;-pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer};+pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer, TranscriptBoundary};pub use session::{StopReason, VoicePhase, VoiceSession};#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rsindex 25d06ae8bc..9c41692e15 100644--- a/rust/crates/ensu/src/voice/rolling.rs+++ b/rust/crates/ensu/src/voice/rolling.rs@@ -31,7 +31,124 @@ pub(crate) struct DecodeWindow {pub(crate) struct RecognitionUpdate {pub commit: Option<(u64, u64, String)>,pub hypothesis: String,- pub covered_through: u64,+ pub provisional: Vec<AlignedWord>,+ pub boundary: TranscriptBoundary,+}++#[derive(Clone, Debug, Default)]+pub(crate) struct TranscriptBoundary {+ pub end: u64,+ anchor: Option<AlignedWord>,+}++impl TranscriptBoundary {+ pub fn start(&self) -> u64 {+ self.anchor.as_ref().map_or(self.end, |anchor| {+ self.end+ .saturating_sub(LOOKBACK)+ .min(anchor.start.saturating_sub(PRE_SPEECH))+ })+ }++ pub fn recorded_update(+ &self,+ words: Vec<AlignedWord>,+ provisional: &[AlignedWord],+ samples: usize,+ final_window: bool,+ ) -> Result<RecognitionUpdate> {+ let end = self.start() + samples as u64;+ let words = absolute_words(words, self.start(), end)?;+ let skip = self.overlap_end(&words)?;+ if words[skip..].is_empty() {+ return Ok(self.pending_update(provisional.to_vec()));+ }+ let words = merge_provisional(provisional, &words[skip..], self.start(), end)?;+ let count = words+ .iter()+ .take_while(|word| {+ word.end+ <= if final_window {+ end+ } else {+ end.saturating_sub(LOOKBACK)+ }+ })+ .count();+ if count > 0 && words[count - 1].end <= self.end {+ return Err(VoiceError::UnstableTranscript);+ }+ let boundary = if count == 0 {+ self.clone()+ } else {+ let anchor = words[count - 1].clone();+ Self {+ end: anchor.end,+ anchor: Some(anchor),+ }+ };+ let commit = (boundary.end > self.end)+ .then(|| (self.end, boundary.end, join_words(&words[..count])));+ let provisional = words[count..].to_vec();+ Ok(RecognitionUpdate {+ commit,+ hypothesis: join_words(&provisional),+ provisional,+ boundary,+ })+ }++ pub fn pending_update(&self, provisional: Vec<AlignedWord>) -> RecognitionUpdate {+ RecognitionUpdate {+ commit: None,+ hypothesis: join_words(&provisional),+ provisional,+ boundary: self.clone(),+ }+ }++ fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {+ let Some(anchor) = &self.anchor else {+ return Ok(0);+ };+ if words.is_empty() {+ return Ok(0);+ }+ let anchor_key = alignment_key(&anchor.text);+ for same_text in [true, false] {+ let mut matches = words.iter().enumerate().filter_map(|(i, word)| {+ let duration = if same_text {+ (word.end - word.start).min(anchor.end - anchor.start)+ } else {+ (word.end - word.start).max(anchor.end - anchor.start)+ };+ let overlap = word+ .end+ .min(anchor.end)+ .saturating_sub(word.start.max(anchor.start));+ (word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE+ && word.end.abs_diff(anchor.end) <= ALIGNMENT_TOLERANCE+ && (overlap * 2 > duration+ || (same_text && duration == 0 && word.start == anchor.start))+ && if same_text {+ alignment_key(&word.text) == anchor_key+ } else {+ word.end <= self.end+ })+ .then_some(i + 1)+ });+ match (matches.next(), matches.next()) {+ (Some(end), None) => return Ok(end),+ (Some(_), Some(_)) => return Err(VoiceError::UnstableTranscript),+ _ => {}+ }+ }+ if words[0].start >= self.end {+ Ok(0)+ } else {+ Err(VoiceError::UnstableTranscript)+ }+ }}struct Region {@@ -145,29 +262,18 @@ impl RollingRecognizer {if self.in_flight != Some((request.id, request.start, request.end, request.final_phrase)) {return Err(VoiceError::StaleEvent);}- let mut current = words;- let mut previous_start = 0;- for word in &mut current {- if word.text.trim().is_empty()- || word.start < previous_start- || word.end < word.start- || word.end > request.end - request.start- {- return Err(VoiceError::UnstableTranscript);- }- previous_start = word.start;- word.start += request.start;- word.end += request.start;- }- let skip = self.overlap_end(¤t)?;+ let current = absolute_words(words, request.start, request.end)?;+ let boundary = self.boundary();+ let skip = boundary.overlap_end(¤t)?;let remaining = ¤t[skip..];- let old_skip = self.overlap_end(&self.previous)?;+ let old_skip = boundary.overlap_end(&self.previous)?;let mut observations = vec![1; current.len()];let agreed_prefix = remaining.iter().zip(&self.previous[old_skip..]).take_while(|(a, b)| {- a.text == b.text && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE+ alignment_key(&a.text) == alignment_key(&b.text)+ && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE}).count();for i in 0..agreed_prefix {@@ -188,8 +294,13 @@ impl RollingRecognizer {} else {agreed};- if request.final_phrase && current.is_empty() {- return Err(VoiceError::IncompleteAudio);+ if request.final_phrase {+ let last_word = remaining.last().ok_or(VoiceError::IncompleteAudio)?;+ if self.regions.front().is_some_and(|region| {+ last_word.end.saturating_add(VOICE_SAMPLE_RATE / 2) < region.last_voice+ }) {+ return Err(VoiceError::IncompleteAudio);+ }}let commit = if commit_count > 0 {let end = if request.final_phrase {@@ -208,7 +319,8 @@ impl RollingRecognizer {} else {None};- let hypothesis = join_words(&remaining[commit_count..]);+ let provisional = remaining[commit_count..].to_vec();+ let hypothesis = join_words(&provisional);self.previous = current;self.observations = observations;self.last_decoded = request.end;@@ -227,11 +339,7 @@ impl RollingRecognizer {if self.covered <= request.start + LOOKBACK {return Err(VoiceError::UnstableTranscript);}- retain_from = self- .covered- .saturating_sub(LOOKBACK)- .min(self.anchor.as_ref().map_or(self.covered, |w| w.start))- .max(request.start);+ retain_from = self.boundary().start().max(request.start);if retain_from <= request.start {return Err(VoiceError::UnstableTranscript);}@@ -247,27 +355,15 @@ impl RollingRecognizer {Ok(RecognitionUpdate {commit,hypothesis,- covered_through: self.covered,+ provisional,+ boundary: self.boundary(),})}- fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {- let Some(anchor) = &self.anchor else {- return Ok(0);- };- if words.is_empty() {- return Ok(0);- }- let anchor_key = alignment_key(&anchor.text);- let mut matches = words.iter().enumerate().filter_map(|(i, word)| {- (alignment_key(&word.text) == anchor_key- && word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE)- .then_some(i + 1)- });- match (matches.next(), matches.next()) {- (Some(end), None) => Ok(end),- (None, None) if words[0].start >= self.covered => Ok(0),- _ => Err(VoiceError::UnstableTranscript),+ fn boundary(&self) -> TranscriptBoundary {+ TranscriptBoundary {+ end: self.covered,+ anchor: self.anchor.clone(),}}@@ -284,6 +380,114 @@ impl RollingRecognizer {}}+fn merge_provisional(+ previous: &[AlignedWord],+ current: &[AlignedWord],+ start: u64,+ end: u64,+) -> Result<Vec<AlignedWord>> {+ let mut paired = vec![None; previous.len()];+ let mut matched = vec![false; current.len()];+ for (old_index, old) in previous.iter().enumerate() {+ if old.start < start || old.end > end {+ continue;+ }+ let key = alignment_key(&old.text);+ let mut candidates = current.iter().enumerate().filter_map(|(index, word)| {+ let same = alignment_key(&word.text) == key;+ let duration = if same {+ (word.end - word.start).min(old.end - old.start)+ } else {+ (word.end - word.start).max(old.end - old.start)+ };+ let overlap = word+ .end+ .min(old.end)+ .saturating_sub(word.start.max(old.start));+ (word.start.abs_diff(old.start) <= ALIGNMENT_TOLERANCE+ && (same || word.end.abs_diff(old.end) <= ALIGNMENT_TOLERANCE)+ && (overlap * 2 > duration || (same && duration == 0 && word.start == old.start)))+ .then_some(index)+ });+ match (candidates.next(), candidates.next()) {+ (Some(index), None) if !matched[index] => {+ matched[index] = true;+ paired[old_index] = Some(index);+ }+ (None, None) => {}+ _ => return Err(VoiceError::UnstableTranscript),+ }+ }+ let anchors = paired.clone();+ for index in 1..previous.len() {+ let (Some(left), Some(right)) = (anchors[index - 1], anchors[index]) else {+ continue;+ };+ if right != left + 1+ || alignment_key(&previous[index - 1].text) != alignment_key(¤t[left].text)+ || alignment_key(&previous[index].text) != alignment_key(¤t[right].text)+ {+ continue;+ }+ let mut old_index = index + 1;+ let mut new_index = right + 1;+ while let (Some(old), Some(word)) = (previous.get(old_index), current.get(new_index)) {+ if old.start < start+ || old.end > end+ || alignment_key(&old.text) != alignment_key(&word.text)+ {+ break;+ }+ if paired[old_index].is_some_and(|paired| paired != new_index)+ || (paired[old_index].is_none() && matched[new_index])+ {+ return Err(VoiceError::UnstableTranscript);+ }+ paired[old_index] = Some(new_index);+ matched[new_index] = true;+ old_index += 1;+ new_index += 1;+ }+ }+ let mut retained = Vec::new();+ for (old, paired) in previous.iter().zip(paired) {+ if paired.is_some() {+ continue;+ }+ if current.iter().any(|word| {+ (word.start < old.end && old.start < word.end)+ || (word.start == old.start && (word.end == word.start || old.end == old.start))+ }) {+ return Err(if old.start < start || old.end > end {+ VoiceError::IncompleteAudio+ } else {+ VoiceError::UnstableTranscript+ });+ }+ retained.push(old.clone());+ }+ retained.extend_from_slice(current);+ retained.sort_by_key(|word| word.start);+ Ok(retained)+}++fn absolute_words(mut words: Vec<AlignedWord>, start: u64, end: u64) -> Result<Vec<AlignedWord>> {+ let mut previous_start = 0;+ for word in &mut words {+ if word.text.trim().is_empty()+ || word.start < previous_start+ || word.end < word.start+ || word.end > end - start+ {+ return Err(VoiceError::UnstableTranscript);+ }+ previous_start = word.start;+ word.start += start;+ word.end += start;+ }+ Ok(words)+}+fn alignment_key(text: &str) -> String {let lexical = text.trim().trim_matches(['.', ',', '!', '?', ':', ';', '…', '。', ',', '!', '?', '؛', '،', '।', '॥', '¿', '¡',@@ -321,9 +525,13 @@ mod tests {fn provisional_revisions_need_repeated_right_context_before_commit() {let mut rolling = RollingRecognizer::default();for index in 0..5 {- rolling.feed(&vec![0.1; 8000], true).unwrap();+ rolling.feed(&vec![0.1; 8000], index == 0).unwrap();let request = rolling.next_window().unwrap().unwrap();- let first = if index < 2 { "Friday" } else { "Saturday" };+ let first = match index {+ 0 | 1 => "Friday",+ 3 => "Saturday,",+ _ => "Saturday",+ };let update = rolling.complete(&request,@@ -342,13 +550,36 @@ mod tests {assert_eq!(update.hypothesis, "at noon");}}+ rolling.finish();+ let request = rolling.next_window().unwrap().unwrap();+ for words in [Vec::new(), vec![aligned("Saturday", 0, 1000)]] {+ assert!(matches!(+ rolling.complete(&request, words),+ Err(VoiceError::IncompleteAudio)+ ));+ assert_eq!(rolling.covered(), 1000);+ assert!(!rolling.is_settled());+ }+ let update = rolling+ .complete(+ &request,+ vec![+ aligned("Saturday", 0, 1000),+ aligned("at", 1000, 2000),+ aligned("noon", 2000, 3000),+ ],+ )+ .unwrap();+ assert_eq!(update.commit.unwrap(), (1000, 40000, "at noon".into()));+ assert!(update.hypothesis.is_empty());+ assert!(rolling.is_settled());}#[test]fn repeated_unfinished_tail_stays_revisable_until_endpoint() {let mut rolling = RollingRecognizer::default();- for _ in 0..6 {- rolling.feed(&vec![0.1; 8000], true).unwrap();+ for index in 0..6 {+ rolling.feed(&vec![0.1; 8000], index == 0).unwrap();let request = rolling.next_window().unwrap().unwrap();let update = rolling.complete(@@ -386,39 +617,252 @@ mod tests {#[test]fn overlap_ignores_boundary_punctuation_but_preserves_acoustic_identity() {- let rolling = RollingRecognizer {+ let boundary = TranscriptBoundary {anchor: Some(aligned("You.", 16000, 20000)),- covered: 20000,- ..RollingRecognizer::default()+ end: 20000,};assert_eq!(- rolling.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),+ boundary.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),Ok(1));assert_eq!(- rolling.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),+ boundary.overlap_end(&[aligned("to", 14000, 18000), aligned("you", 16000, 20000)]),+ Ok(2)+ );+ assert_eq!(+ boundary.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),Err(VoiceError::UnstableTranscript));assert_eq!(- rolling.overlap_end(&[aligned("you", 12000, 20000)]),+ boundary.overlap_end(&[aligned("you", 12000, 20000)]),Err(VoiceError::UnstableTranscript));+ assert_eq!(boundary.overlap_end(&[aligned("not", 16000, 20000)]), Ok(1));assert_eq!(- rolling.overlap_end(&[aligned("not", 16000, 20000)]),+ boundary.overlap_end(&[aligned("to", 14000, 18000)]),+ Err(VoiceError::UnstableTranscript)+ );+ let repeated = [aligned("no", 0, 1600), aligned("no", 1600, 3200)];+ assert!(matches!(+ merge_provisional(&repeated, &[aligned("no", 0, 3200)], 0, 40000),+ Err(VoiceError::UnstableTranscript)+ ));+ assert_eq!(+ join_words(+ &merge_provisional(+ &[aligned("no", 0, 1600)],+ &[aligned("no", 1600, 3200)],+ 0,+ 40000+ )+ .unwrap()+ ),+ "no no"+ );+ assert!(matches!(+ merge_provisional(+ &[aligned("a", 16000, 16000)],+ &[aligned("the", 16000, 16000)],+ 0,+ 40000+ ),Err(VoiceError::UnstableTranscript)+ ));+ let retained = merge_provisional(+ &[+ aligned("earlier", 16000, 20000),+ aligned("later", 100000, 104000),+ ],+ &[aligned("corrected", 40000, 44000)],+ 0,+ 80000,+ )+ .unwrap();+ assert_eq!(join_words(&retained), "earlier corrected later");+ assert!(matches!(+ merge_provisional(+ &[aligned("hello", 100, 300)],+ &[aligned("hello", 100, 200)],+ 0,+ 200,+ ),+ Err(VoiceError::IncompleteAudio)+ ));+ let compressed = [+ aligned("campuses", 464320, 473280),+ aligned("easier", 473280, 478400),+ aligned("to", 478400, 479680),+ aligned("reach", 479680, 482240),+ aligned("than", 482240, 483040),+ ];+ let revised = [+ aligned("campuses,", 464480, 474720),+ aligned("easier", 474720, 481120),+ aligned("to", 481120, 483680),+ aligned("reach", 483680, 487520),+ aligned("than", 487520, 490080),+ ];+ assert_eq!(+ join_words(&merge_provisional(&compressed, &revised, 440320, 581120).unwrap()),+ "campuses, easier to reach than"+ );+ let omitted = [&revised[..3], &revised[4..]].concat();+ assert!(+ merge_provisional(&compressed, &omitted, 440320, 581120)+ .map_or(true, |words| join_words(&words).contains("reach")));assert_ne!(alignment_key("don't"), alignment_key("do"));assert_ne!(alignment_key("कि"), alignment_key("क"));assert_ne!(alignment_key("-5"), alignment_key("5"));- let rolling = RollingRecognizer {+ let boundary = TranscriptBoundary {+ anchor: Some(aligned("two", 16000, 18560)),+ end: 18560,+ };+ for revised in ["two", "to"] {+ assert_eq!(+ boundary+ .recorded_update(+ vec![+ aligned(revised, 16000, 18560),+ aligned("two", 18560, 21120),+ aligned("people", 21120, 25000),+ ],+ &[],+ 32000,+ true,+ )+ .unwrap()+ .commit+ .unwrap()+ .2,+ "two people"+ );+ }+ let boundary = TranscriptBoundary {+ anchor: Some(aligned("a", 16000, 16000)),+ end: 16000,+ };+ assert_eq!(boundary.overlap_end(&[aligned("a", 16000, 16000)]), Ok(1));+ assert_eq!(boundary.overlap_end(&[aligned("the", 16000, 16000)]), Ok(0));+ assert_eq!(+ boundary.overlap_end(&[aligned("a", 16000, 16000), aligned("a", 16000, 16000)]),+ Err(VoiceError::UnstableTranscript)+ );+ let boundary = TranscriptBoundary {+ anchor: Some(aligned("going", 16000, 21000)),+ end: 21000,+ };+ assert_eq!(+ boundary.overlap_end(&[aligned("gonna", 16000, 23000)]),+ Err(VoiceError::UnstableTranscript)+ );+ assert_eq!(+ boundary.overlap_end(&[aligned("going", 16000, 23000)]),+ Ok(1)+ );+ let boundary = TranscriptBoundary {+ anchor: Some(aligned("are", 338400, 339200)),+ end: 339200,+ };+ assert_eq!(+ boundary.overlap_end(&[aligned("are", 338080, 340640)]),+ Ok(1)+ );+ assert_eq!(+ boundary.overlap_end(&[aligned("were", 338080, 340640)]),+ Err(VoiceError::UnstableTranscript)+ );+ assert_eq!(+ boundary.overlap_end(&[+ aligned("are", 338080, 340640),+ aligned("are", 338400, 339200),+ ]),+ Err(VoiceError::UnstableTranscript)+ );+ let boundary = TranscriptBoundary {anchor: Some(aligned("Café!", 16000, 20000)),- covered: 20000,- ..RollingRecognizer::default()+ end: 20000,+ };+ assert_eq!(+ boundary.overlap_end(&[aligned("café", 16000, 20000)]),+ Ok(1)+ );+ assert_eq!(+ boundary.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),+ Ok(1)+ );+ let boundary = TranscriptBoundary {+ end: 56000,+ anchor: Some(aligned("You.", 50000, 56000)),};- assert_eq!(rolling.overlap_end(&[aligned("café", 16000, 20000)]), Ok(1));+ assert_eq!(boundary.start(), 32000);assert_eq!(- rolling.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),+ boundary+ .recorded_update(+ vec![+ aligned("before", 0, 8000),+ aligned("you,", 18000, 24000),+ aligned("you", 26000, 30000),+ aligned("again", 30000, 34000),+ ],+ &[],+ 40000,+ true,+ )+ .unwrap()+ .commit+ .unwrap()+ .2,+ "you again"+ );+ assert!(matches!(+ boundary.recorded_update(vec![aligned("different", 12000, 24000)], &[], 40000, true),+ Err(VoiceError::UnstableTranscript)+ ));+ assert!(matches!(+ boundary.recorded_update(vec![aligned("you", 18000, 50000)], &[], 40000, true),Err(VoiceError::UnstableTranscript)+ ));+ let provisional = [aligned("you", 58000, 62000), aligned("again", 62000, 66000)];+ for words in [Vec::new(), vec![aligned("you", 18000, 24000)]] {+ let update = boundary+ .recorded_update(words, &provisional, 40000, true)+ .unwrap();+ assert!(update.commit.is_none());+ assert_eq!(update.boundary.end, boundary.end);+ assert_eq!(update.hypothesis, "you again");+ assert_eq!(join_words(&update.provisional), "you again");+ }+ assert_eq!(+ TranscriptBoundary {+ end: 56000,+ anchor: None+ }+ .start(),+ 56000+ );+ let boundary = TranscriptBoundary {+ end: 403520,+ anchor: Some(aligned("Americans", 379200, 403520)),+ };+ assert_eq!(boundary.start(), 363200);+ assert_eq!(+ boundary+ .recorded_update(+ vec![+ aligned("America,", 18400, 40160),+ aligned("ask", 40160, 51680),+ aligned("Americans", 70000, 94000),+ ],+ &[],+ 128000,+ true,+ )+ .unwrap()+ .commit+ .unwrap()+ .2,+ "ask Americans");}@@ -490,22 +934,32 @@ mod tests {#[test]fn failed_alignment_keeps_audio_and_does_not_claim_completion() {- let mut rolling = RollingRecognizer::default();- rolling.feed(&vec![0.1; 8000], true).unwrap();- let request = rolling.next_window().unwrap().unwrap();- assert!(- rolling- .complete(- &request,- vec![AlignedWord {- text: "bad".into(),- start: 0,- end: 9000- }]- )- .is_err()- );- assert_eq!(rolling.retained_samples(), 8000);- assert_eq!(rolling.covered(), 0);+ for (samples, finished, word_end, error) in [+ (8000, false, 9000, VoiceError::UnstableTranscript),+ (32000, true, 8000, VoiceError::IncompleteAudio),+ ] {+ let mut rolling = RollingRecognizer::default();+ rolling.feed(&vec![0.1; samples], true).unwrap();+ if finished {+ rolling.finish();+ }+ let request = rolling.next_window().unwrap().unwrap();+ assert_eq!(+ rolling+ .complete(+ &request,+ vec![AlignedWord {+ text: "prefix".into(),+ start: 0,+ end: word_end,+ }]+ )+ .unwrap_err(),+ error+ );+ assert_eq!(rolling.retained_samples(), samples);+ assert_eq!(rolling.covered(), 0);+ assert!(!rolling.is_settled());+ }}}