π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Harden mobile dictation lifecycle and bounded transcription
setalpc73247724d25
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex f5822f52d1..537b82b3f6 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -10,6 +10,7 @@ import io.ente.ensu.chat.AttachmentStoreActionsimport io.ente.ensu.chat.ChatMessageimport io.ente.ensu.chat.ChatRepositoryimport io.ente.ensu.chat.ChatStoreActions+import io.ente.ensu.chat.VoiceInputFailureimport io.ente.ensu.chat.VoiceTranscriptionControllerimport io.ente.ensu.device.AndroidDeviceCapabilityProviderimport io.ente.ensu.device.ChatDeviceCapability@@ -91,6 +92,12 @@ class AppStore(private var composerRevision = 0uLprivate var emptyComposerId = UUID.randomUUID().toString()++ private fun invalidateComposer() {+ composerRevision+++ voiceController.composerCaret = null+ }+internal val voiceController: VoiceTranscriptionController =VoiceTranscriptionController(context.applicationContext,@@ -167,7 +174,7 @@ class AppStore(fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)fun handleMemoryPressure() {- voiceController.interrupt("Voice input paused because memory is low.")+ voiceController.interrupt(VoiceInputFailure.MemoryUnavailable)chatActions.cancelSessionSummary()modelSettingsActions.handleMemoryPressure()}@@ -180,6 +187,10 @@ class AppStore(fun refreshDeviceCapability(scope: CoroutineScope? = null) {val capability = deviceCapabilityProvider.chatCapability()val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemory+ if (unsupported) {+ voiceController.discard()+ invalidateComposer()+ }_state.value =_state.value.copy(chat =@@ -234,23 +245,30 @@ class AppStore()}- fun createNewSession(): String = chatActions.createNewSession()+ fun createNewSession(): String {+ voiceController.discard()+ invalidateComposer()+ return chatActions.createNewSession()+ }fun startNewSessionDraft() {voiceController.discard()emptyComposerId = UUID.randomUUID().toString()- composerRevision+++ invalidateComposer()chatActions.startNewSessionDraft()}fun selectSession(sessionId: String) {voiceController.discard()- composerRevision+++ invalidateComposer()chatActions.selectSession(sessionId)}fun deleteSession(sessionId: String) {- if (sessionId == _state.value.chat.currentSessionId) voiceController.discard()+ if (sessionId == _state.value.chat.currentSessionId) {+ voiceController.discard()+ invalidateComposer()+ }chatActions.deleteSession(sessionId)}@@ -259,8 +277,7 @@ class AppStore(fun updateMessageText(value: String) {if (value != _state.value.chat.messageText) {- composerRevision++- voiceController.composerCaret = null+ invalidateComposer()}chatActions.updateMessageText(value)}@@ -270,11 +287,14 @@ class AppStore(fun beginEditing(messageId: String) {voiceController.discard()- composerRevision+++ invalidateComposer()chatActions.beginEditing(messageId)}- fun cancelEditing() = chatActions.cancelEditing()+ fun cancelEditing() {+ invalidateComposer()+ chatActions.cancelEditing()+ }fun setAttachmentProcessing(isProcessing: Boolean) =attachmentActions.setAttachmentProcessing(isProcessing)@@ -319,8 +339,7 @@ class AppStore(fun sendMessage() {if (voiceController.active) return- voiceController.composerCaret = null- composerRevision+++ invalidateComposer()chatActions.sendMessage()}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.ktindex 9f2719038f..e8ea62c6ae 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt@@ -16,7 +16,10 @@ import io.ente.ensu.bindings.needsAssetMigrationimport io.ente.ensu.bindings.reconcileKnowledgePackimport io.ente.ensu.bindings.uniffiEnsureInitializedimport java.io.File+import kotlinx.coroutines.CoroutineStartimport kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.awaitCancellation+import kotlinx.coroutines.launchimport kotlinx.coroutines.withContextclass AssetStore(context: Context) {@@ -90,6 +93,14 @@ class AssetStore(context: Context) {val token = CancellationToken()val lease = AssetDownloadJobService.begin { token.cancel() }+ val cancellation =+ launch(Dispatchers.Default, start = CoroutineStart.UNDISPATCHED) {+ try {+ awaitCancellation()+ } finally {+ token.cancel()+ }+ }try {core.download(assets,@@ -107,7 +118,7 @@ class AssetStore(context: Context) {token,)} finally {- token.cancel()+ cancellation.cancel()AssetDownloadJobService.end(lease)}}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.ktindex 19b5e4ebce..5d779963c1 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt@@ -14,7 +14,6 @@ internal fun captureMicrophoneAudio(if (isStopped()) returnthrow IOException("Microphone read failed")}- // Stop changes the route too. Preserve a successful final read while draining.if (isInterrupted() && !isStopped()) throw IOException("Microphone interrupted")if (!deliver(count)) return}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex a0fcd2ee21..55a0c40e4e 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -2,7 +2,6 @@ package io.ente.ensu.chatimport android.annotation.SuppressLintimport android.content.Context-import android.content.pm.ApplicationInfoimport android.media.AudioFormatimport android.media.AudioManagerimport android.media.AudioRecord@@ -11,6 +10,7 @@ import androidx.compose.runtime.getValueimport androidx.compose.runtime.mutableStateOfimport androidx.compose.runtime.setValueimport io.ente.ensu.assets.AssetStore+import io.ente.ensu.bindings.AssetDownloadExceptionimport io.ente.ensu.bindings.DictationExceptionimport io.ente.ensu.bindings.DictationFeedResultimport io.ente.ensu.bindings.DictationInsertion@@ -20,11 +20,12 @@ import io.ente.ensu.bindings.DictationSnapshotimport io.ente.ensu.bindings.DictationStopReasonimport io.ente.ensu.bindings.DictationStreamimport io.ente.ensu.bindings.DictationStreamCallback+import io.ente.ensu.bindings.supportsLiveVoiceimport io.ente.ensu.bindings.transcriptionModelAssetimport io.ente.ensu.bindings.voiceActivityModelAssetimport io.ente.ensu.device.AndroidDeviceCapabilityProvider-import io.ente.ensu.device.supportsLiveVoiceimport io.ente.ensu.llm.LlmProvider+import io.ente.ensu.llm.ModelMemoryUnavailableimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.logging.LogLevelimport java.io.IOException@@ -46,6 +47,36 @@ import kotlinx.coroutines.delayimport kotlinx.coroutines.launchimport kotlinx.coroutines.withContext+internal enum class VoiceInputFailure(val message: String) {+ Permission("Microphone permission is required. Enable it in Settings."),+ NoSpeech("No speech detected. Try again."),+ Download("Voice model download failed. Check your connection and try again."),+ ModelUnavailable("Voice model could not be loaded. Try again."),+ StorageFull("Not enough storage space. Free up space and try again."),+ FileAccess("Could not read or save voice data. Check storage and try again."),+ MemoryUnavailable("Not enough free memory. Close other apps and try again."),+ MicrophoneUnavailable("Could not start the microphone. Tap the microphone to try again."),+ Interrupted("Voice input was interrupted. Review the available text and try again."),+ ProcessingFailed("Could not transcribe voice input. Review the available text and try again."),+}++internal fun voiceInputFailure(error: Exception): VoiceInputFailure =+ when (error) {+ is ModelMemoryUnavailable -> VoiceInputFailure.MemoryUnavailable+ is AssetDownloadException.StorageFull,+ is DictationException.StorageFull -> VoiceInputFailure.StorageFull+ is AssetDownloadException.Cancelled,+ is DictationException.Cancelled,+ is DictationException.InvalidAudio -> VoiceInputFailure.Interrupted+ is AssetDownloadException.Io,+ is DictationException.Io,+ is IOException -> VoiceInputFailure.FileAccess+ is AssetDownloadException -> VoiceInputFailure.Download+ is DictationException.NotDownloaded,+ is DictationException.VadNotDownloaded -> VoiceInputFailure.ModelUnavailable+ else -> VoiceInputFailure.ProcessingFailed+ }+internal sealed interface VoiceInputState {data object Idle : VoiceInputState@@ -57,7 +88,7 @@ internal sealed interface VoiceInputState {data object Transcribing : VoiceInputState- data class Error(val message: String) : VoiceInputState+ data class Error(val reason: VoiceInputFailure) : VoiceInputState}internal val VoiceInputState.isRecording: Boolean@@ -71,20 +102,23 @@ internal class VoiceTranscriptionController(private val onVoiceInputJob: (Job) -> Unit,private val onInsert: (DictationInsertion, String) -> Boolean,) {- private val diagnosticsEnabled =- context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0-- private fun diagnostic(event: String) {- if (diagnosticsEnabled) {- logRepository.log(LogLevel.Info, event, tag = "EnsuVoice")- }- }+ private fun logFailure(error: Exception) =+ logRepository.log(+ LogLevel.Error,+ "Voice input failed",+ tag = "EnsuVoice",+ throwable = error,+ )private val audioManager = context.getSystemService(AudioManager::class.java)private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)private val usesLiveDictation =supportsLiveVoice(- AndroidDeviceCapabilityProvider(context).chatCapability().totalMemoryBytes+ AndroidDeviceCapabilityProvider(context)+ .chatCapability()+ .totalMemoryBytes+ ?.takeIf { it > 0 }+ ?.toULong())private val modelAssets =listOf(@@ -119,7 +153,7 @@ internal class VoiceTranscriptionController(var snapshot: DictationSnapshot? by mutableStateOf(null)private set- private var notice: String? = null+ private var failure: VoiceInputFailure? = nullvar audioLevels: List<Float> by mutableStateOf(List(24) { 0f })private set@@ -141,7 +175,7 @@ internal class VoiceTranscriptionController(this.insertion = insertionsession = DictationSession(live = usesLiveDictation)snapshot = session?.snapshot()- notice = null+ failure = nullaudioLevels = List(24) { 0f }active = truestate = VoiceInputState.Idle@@ -166,8 +200,7 @@ internal class VoiceTranscriptionController(if (foreground) record() else startOnForeground = true} else {discard()- state =- VoiceInputState.Error("Microphone permission is required. Enable it in Settings.")+ state = VoiceInputState.Error(VoiceInputFailure.Permission)}}@@ -184,7 +217,7 @@ internal class VoiceTranscriptionController(fun record() {val owner = session ?: returnif (!active || !foreground || hasActiveWork || attemptFinished) return- notice = null+ failure = nullstate = VoiceInputState.Preparingval token = generationhasActiveWork = true@@ -223,18 +256,29 @@ internal class VoiceTranscriptionController(audioLevels = audioLevels.drop(1) + level}},- ) {+ ) { error ->scope.launch {- if (token == generation)- interrupt(- "Microphone interrupted. Review the text before continuing."- )+ if (token == generation) {+ logFailure(error)+ interrupt()+ }}}capture = microphoneownedCapture = microphonecoroutineScope {- val hardwareReady = async(Dispatchers.IO) { microphone.prepare() }+ val hardwareReady =+ async(Dispatchers.IO) {+ try {+ microphone.prepare()+ } catch (error: Exception) {+ withContext(Dispatchers.Main.immediate) {+ if (token == generation && failure == null)+ failure = VoiceInputFailure.MicrophoneUnavailable+ }+ throw error+ }+ }val callback =object : DictationStreamCallback {override fun onReady() {@@ -254,13 +298,17 @@ internal class VoiceTranscriptionController(microphone.start(scope)state = VoiceInputState.RecordingaudioLevels = List(24) { 0f }- notice = null+ failure = nullrefresh(owner, token)- } catch (_: Exception) {- if (token == generation)+ } catch (error: CancellationException) {+ throw error+ } catch (error: Exception) {+ if (token == generation) {+ logFailure(error)interrupt(- "Could not start the microphone. Tap the microphone to start again."+ VoiceInputFailure.MicrophoneUnavailable)+ }}}}@@ -279,16 +327,18 @@ internal class VoiceTranscriptionController(llmProvider.runLiveVoice(activeStream, callback)}} catch (error: CancellationException) {+ if (token == generation && failure == null)+ failure = VoiceInputFailure.Interruptedthrow error} catch (error: Exception) {ownedStream?.let { runCatching { it.abandon(DictationStopReason.FAILURE) } }- diagnostic(- if (error is DictationException.InvalidOperation)- "failure ${error.detail}"- else "failure ${error.javaClass.simpleName}"+ if (+ error !is AssetDownloadException.Cancelled &&+ error !is DictationException.Cancelled)- if (token == generation && notice == null) {- notice = "Voice input stopped. Review the available text or try again."+ logFailure(error)+ if (token == generation && failure == null) {+ failure = voiceInputFailure(error)}} finally {withContext(NonCancellable) { ownedCapture?.stopAndDrain() }@@ -320,8 +370,8 @@ internal class VoiceTranscriptionController(val completedState =when {added && !value.incomplete -> VoiceInputState.Idle- text.isBlank() -> VoiceInputState.Error(notice ?: "No speech detected.")- else -> VoiceInputState.Error("Voice input stopped.")+ text.isBlank() -> VoiceInputState.Error(failure ?: VoiceInputFailure.NoSpeech)+ else -> VoiceInputState.Error(failure ?: VoiceInputFailure.Interrupted)}active = falseisFinishingSlow = false@@ -374,19 +424,19 @@ internal class VoiceTranscriptionController(}}- fun interrupt(message: String = "Voice input stopped.") {+ fun interrupt(reason: VoiceInputFailure = VoiceInputFailure.Interrupted) {if (!active || attemptFinished) returnif (!hasActiveWork && stream == null) {discard()return}+ failure = reason+ state = VoiceInputState.Transcribingcapture?.stop()runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }work?.cancel()finishingNoticeJob?.cancel()isFinishingSlow = false- notice = message- state = VoiceInputState.Transcribing}fun discard() {@@ -418,7 +468,7 @@ private class LiveMicrophone(private val stream: DictationStream,private val audioManager: AudioManager,private val onLevel: (Float) -> Unit,- private val onInterrupted: () -> Unit,+ private val onInterrupted: (Exception) -> Unit,) {private val stopped = AtomicBoolean(false)@Volatile private var recorder: AudioRecord? = null@@ -501,10 +551,10 @@ private class LiveMicrophone()stream.feed(buffer) == DictationFeedResult.ACCEPTED}- } catch (_: Exception) {+ } catch (error: Exception) {if (!stopped.get()) {runCatching { stream.abandon(DictationStopReason.FAILURE) }- onInterrupted()+ onInterrupted(error)}} finally {stop()@@ -517,7 +567,6 @@ private class LiveMicrophone(suspend fun stopAndDrain() {stop()readJob?.join()- // A reader canceled before dispatch never enters its cleanup block.recorder?.release()recorder = null}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.ktindex 4974e98aa2..dd79663204 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt@@ -56,8 +56,7 @@ import io.ente.ensu.designsystem.HugeIconsinternal fun VoiceInputControls(controller: VoiceTranscriptionController) {val state = controller.stateif (!controller.active && state !is VoiceInputState.Error) return- val permissionNeeded =- (state as? VoiceInputState.Error)?.message?.contains("permission") == true+ val permissionNeeded = (state as? VoiceInputState.Error)?.reason == VoiceInputFailure.Permissionval context = LocalContext.currentval label =when (state) {@@ -69,9 +68,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {VoiceInputState.Recording ->if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"else "Listening"- is VoiceInputState.Error ->- if (permissionNeeded) "Microphone access"- else if (state.message == "No speech detected.") "No speech" else "Stopped"+ is VoiceInputState.Error -> state.reason.message}Row(Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.ktindex 832c392269..6bbb678100 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt@@ -2,9 +2,6 @@ package io.ente.ensu.deviceconst val CHAT_MIN_RAM_BYTES: Long = 1_600_000_000L-internal fun supportsLiveVoice(totalMemoryBytes: Long?): Boolean =- totalMemoryBytes != null && totalMemoryBytes > 4L * 1024 * 1024 * 1024-internal fun chatDeviceCapability(totalMemoryBytes: Long): ChatDeviceCapability =when {totalMemoryBytes <= 0 -> ChatDeviceCapability.Unknowndiff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.ktindex aeebb06bd9..82fbceeeea 100644--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt@@ -12,7 +12,6 @@ class MicrophoneCaptureTest {isStopped = { stopped },read = { 3200 },isInterrupted = {- // AudioRecord.stop() clears the route after a successful read.stopped = truetrue},diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.ktindex 5ba9a19595..d1480cefd7 100644--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt@@ -7,15 +7,6 @@ import org.junit.Assert.assertTrueimport org.junit.Testclass ChatDeviceCapabilityTest {- @Test- fun liveVoiceRequiresMoreThanFourGiB() {- val limit = 4L * 1024 * 1024 * 1024- for (bytes in listOf(null, -1L, 0L, 4_000_000_000L, limit)) {- assertFalse(supportsLiveVoice(bytes))- }- for (bytes in listOf(limit + 1, 8_000_000_000L)) assertTrue(supportsLiveVoice(bytes))- }-@Testfun minimumRamBlocksOnlyKnownReadingsBelowTheFloor() {val belowMinimum = chatDeviceCapability(1_599_999_999L)diff --git a/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj b/apple/apps/ensu/Ensu.xcodeproj/project.pbxprojindex 85e1b32b02..a18b68369f 100644--- a/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj+++ b/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj@@ -7,6 +7,7 @@objects = {/* Begin PBXBuildFile section */+ A70000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A70000022F11111100AA0001; };A50000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A50000022F11111100AA0001; };A40000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A40000022F11111100AA0001; };A30000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A30000022F11111100AA0001; };@@ -29,6 +30,7 @@/* End PBXContainerItemProxy section *//* Begin PBXFileReference section */+ A70000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceCaptureDeliveryTests.swift; sourceTree = "<group>"; };A50000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KnowledgePreferencesTests.swift; sourceTree = "<group>"; };A40000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelMemoryCoordinationTests.swift; sourceTree = "<group>"; };A30000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageMarkdownTests.swift; sourceTree = "<group>"; };@@ -85,6 +87,7 @@isa = PBXGroup;children = (A10000022F11111100AA0001 /* LlmIntegrationTests.swift */,+ A70000022F11111100AA0001,A40000022F11111100AA0001,A50000022F11111100AA0001,A20000022F11111100AA0001,@@ -256,6 +259,7 @@buildActionMask = 2147483647;files = (A10000012F11111100AA0001 /* LlmIntegrationTests.swift in Sources */,+ A70000012F11111100AA0001,A40000012F11111100AA0001,A50000012F11111100AA0001,A20000012F11111100AA0001,diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex d7711f92f7..fe2100e4fd 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -272,7 +272,7 @@ final class ChatViewModel: ObservableObject {}func handleMemoryPressure() {- voiceTranscriber.interrupt(message: "Voice input paused because memory is low.")+ voiceTranscriber.interrupt(reason: .memoryUnavailable)sessionSummaryTask?.cancel()warmupSuppressed = truecancelChatWarmup()diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swiftindex 260fb7715c..021cfc7f05 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift@@ -11,35 +11,48 @@ struct VoiceInputControls: View {case .preparing: return "Preparing"case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"- case .error(let message):- if message.contains("permission") { return "Microphone access" }- return message == "No speech detected." ? "No speech" : "Stopped"+ case .error(.permission): return "Microphone access"+ case .error(.noSpeech): return "No speech"+ case .error: return "Stopped"case .idle: return "Preparing"}}+ private var failure: VoiceInputFailure? {+ if case .error(let failure) = service.state { return failure }+ return service.notice+ }+var body: some View {if service.active || service.state.isError {- HStack(spacing: 12) {- Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)- .accessibilityHint(service.notice ?? "")- .onTapGesture {- if case .error(let message) = service.state,- message.contains("permission"),- let url = URL(string: UIApplication.openSettingsURLString)- {- openURL(url)- }+ VStack(alignment: .leading, spacing: EnsuSpacing.xs) {+ HStack(spacing: 12) {+ Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)+ if service.state.isRecording {+ waveform.frame(height: 24)+ } else {+ Spacer(minLength: 0)+ }+ if service.active, !service.state.isRecording {+ Button("Cancel", action: service.cancel)+ .font(EnsuTypography.small)+ .foregroundStyle(EnsuColor.textPrimary)}- if service.state.isRecording {- waveform.frame(height: 24)- } else {- Spacer(minLength: 0)}- if service.active, !service.state.isRecording {- Button("Cancel", action: service.cancel)+ if let failure {+ Text(failure.message)+ .font(EnsuTypography.small)+ .foregroundStyle(EnsuColor.textMuted)+ .fixedSize(horizontal: false, vertical: true)+ if failure == .permission {+ Button("Open Settings") {+ if let url = URL(string: UIApplication.openSettingsURLString) {+ openURL(url)+ }+ }.font(EnsuTypography.small).foregroundStyle(EnsuColor.textPrimary)+ }}}}diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 897e4b98e3..961975aa3d 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -2,13 +2,69 @@ import Foundationimport AVFoundationimport Combine+enum VoiceInputFailure: Equatable {+ case permission, microphone, noSpeech, interrupted, captureInterrupted+ case modelUnavailable, memoryUnavailable, storageFull, download, fileAccess, transcription+ case insertion++ var message: String {+ switch self {+ case .permission:+ return "Microphone permission is required. Enable it in Settings."+ case .microphone:+ return "Could not start the microphone. Tap the microphone to try again."+ case .noSpeech:+ return "No speech detected. Tap the microphone to try again."+ case .interrupted:+ return "Voice input stopped. Review the available text or try again."+ case .captureInterrupted:+ return "Audio capture interrupted. Some words may be missing; review the text."+ case .modelUnavailable:+ return "The voice model is unavailable. Tap the microphone to download it again."+ case .memoryUnavailable:+ return "Not enough free memory for voice input. Close other apps and try again."+ case .storageFull:+ return "Not enough storage for voice input. Free up space and try again."+ case .download:+ return "Could not download the voice model. Check your connection and try again."+ case .fileAccess:+ return "Could not access voice model files. Try again."+ case .transcription:+ return "Could not transcribe voice input. Review the available text or try again."+ case .insertion:+ return "The draft changed before voice input finished. Tap the microphone to try again."+ }+ }++ static func classify(_ error: Error) -> Self? {+ switch error {+ case is CancellationError, AssetDownloadError.Cancelled, DictationError.Cancelled:+ return nil+ case is ModelMemoryUnavailable:+ return .memoryUnavailable+ case DictationError.NotDownloaded, DictationError.VadNotDownloaded:+ return .modelUnavailable+ case AssetDownloadError.StorageFull, DictationError.StorageFull:+ return .storageFull+ case AssetDownloadError.Io, DictationError.Io:+ return .fileAccess+ case is AssetDownloadError:+ return .download+ case DictationError.InvalidAudio:+ return .captureInterrupted+ default:+ return .transcription+ }+ }+}+enum VoiceInputState: Equatable {case idlecase preparingcase recordingcase downloading(percent: Int?)case transcribing- case error(String)+ case error(VoiceInputFailure)var isRecording: Bool {self == .recording@@ -30,7 +86,7 @@ final class VoiceTranscriptionService: ObservableObject {}@Published private(set) var snapshot: DictationSnapshot?@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)- @Published private(set) var notice: String?+ @Published private(set) var notice: VoiceInputFailure?@Published private(set) var state: VoiceInputState = .idle@Published private(set) var isFinishingSlow = falsevar onActivityChanged: (@MainActor () -> Void)?@@ -39,6 +95,7 @@ final class VoiceTranscriptionService: ObservableObject {private let assetStore: AssetStoreprivate let modelAssets: [Asset]private let usesLiveDictation: Bool+ private let logger = EnsuLogging.shared.logger("EnsuVoice")private var session: DictationSession?private var stream: DictationStream?private var recorder: LivePcmRecorder?@@ -56,14 +113,15 @@ final class VoiceTranscriptionService: ObservableObject {}private func diagnostic(_ event: String) {#if DEBUG- EnsuLogging.shared.logger("EnsuVoice").info(event)+ logger.info(event)#endif}init(provider: LlmProvider, assetStore: AssetStore) {self.provider = providerself.assetStore = assetStore- usesLiveDictation = supportsLiveVoice()+ usesLiveDictation = supportsLiveVoice(+ totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)modelAssets = [transcriptionModelAsset(),voiceActivityModelAsset(),@@ -149,15 +207,14 @@ final class VoiceTranscriptionService: ObservableObject {if self.foreground { self.record() } else { self.startOnForeground = true }} else {self.cancel()- self.state = .error(- "Microphone permission is required. Enable it in Settings.")+ self.state = .error(.permission)}}}returncase .denied:cancel()- state = .error("Microphone permission is required. Enable it in Settings.")+ state = .error(.permission)returncase .granted: break@unknown default: return@@ -195,7 +252,14 @@ final class VoiceTranscriptionService: ObservableObject {try Task.checkCancellation()guard self.generation == token, self.foreground else { return }self.state = .preparing- let recorder = try LivePcmRecorder()+ let recorder: LivePcmRecorder+ do {+ recorder = try LivePcmRecorder()+ } catch {+ self.logger.error("Could not prepare the microphone", error)+ self.notice = .microphone+ return+ }self.recorder = recorderlet sampleRate = recorder.sampleRatelet stream = try DictationStream(@@ -212,13 +276,15 @@ final class VoiceTranscriptionService: ObservableObject {}}) {- [weak self] in+ [weak self] error inTask { @MainActor [weak self] inguard let self, self.generation == token else { return }- self.interrupt(- message:- "Audio capture interrupted. Some words may be missing; review the text."- )+ if let error {+ self.logger.error("Audio capture failed", error)+ } else {+ self.logger.warning("Audio capture exceeded its buffer limit")+ }+ self.interrupt(reason: .captureInterrupted)}}let callback = LiveVoiceCallback(@@ -236,10 +302,8 @@ final class VoiceTranscriptionService: ObservableObject {self.notice = nilself.refresh(owner, token: token)} catch {- self.interrupt(- message:- "Could not start the microphone. Tap the microphone to start again."- )+ self.logger.error("Could not start the microphone", error)+ self.interrupt(reason: .microphone)}}},@@ -257,16 +321,11 @@ final class VoiceTranscriptionService: ObservableObject {})try await self.provider.runLiveVoice(stream: stream, callback: callback)- } catch is CancellationError {} catch {- if let error = error as? DictationError, case .InvalidOperation(let detail) = error- {- self.diagnostic("failure \(detail)")- } else {- self.diagnostic("failure \(type(of: error))")- }+ let failure = VoiceInputFailure.classify(error)+ if failure != nil { self.logger.error("Voice input failed", error) }if self.generation == token, self.notice == nil {- self.notice = "Voice input stopped. Review the available text or try again."+ self.notice = failure ?? .interrupted}}}@@ -286,10 +345,9 @@ final class VoiceTranscriptionService: ObservableObject {let added = hasText && insertion.map { onInsert?($0, text) == true } == truelet completedState: VoiceInputStateif added {- completedState = value.incomplete ? .error("Voice input stopped.") : .idle+ completedState = value.incomplete ? .error(notice ?? .interrupted) : .idle} else {- completedState = .error(- hasText ? "Voice input stopped." : notice ?? "No speech detected.")+ completedState = .error(hasText ? .insertion : notice ?? .noSpeech)}active = falseisFinishingSlow = false@@ -327,7 +385,7 @@ final class VoiceTranscriptionService: ObservableObject {if recorder == nil { try? stream?.stop(reason: reason) }}- func interrupt(message: String = "Voice input stopped.") {+ func interrupt(reason: VoiceInputFailure = .interrupted) {guard active, !attemptFinished else { return }if !hasActiveTasks, stream == nil {cancel()@@ -338,7 +396,7 @@ final class VoiceTranscriptionService: ObservableObject {task?.cancel()finishingNoticeTask?.cancel()isFinishingSlow = false- notice = message+ notice = reasonstate = .transcribing}@@ -372,7 +430,9 @@ private final class LiveVoiceCallback: DictationStreamCallback, Sendable {change: @escaping @Sendable () -> Void,closed: @escaping @Sendable () -> Void) {- self.ready = ready; self.change = change; self.closed = closed+ self.ready = ready+ self.change = change+ self.closed = closed}func onReady() { ready() }func onChange() { change() }@@ -416,7 +476,10 @@ private final class LivePcmRecorder {}func stop(afterDelivery: (@Sendable () -> Void)? = nil) {- if installed { engine.inputNode.removeTap(onBus: 0); installed = false }+ if installed {+ engine.inputNode.removeTap(onBus: 0)+ installed = false+ }delivery?.close()engine.stop()try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)@@ -431,7 +494,7 @@ private final class VoiceNotificationTokens: @unchecked Sendable {deinit { for value in values { NotificationCenter.default.removeObserver(value) } }}-private final class LiveCaptureDelivery: @unchecked Sendable {+final class LiveCaptureDelivery: @unchecked Sendable {private let queue = DispatchQueue(label: "io.ente.ensu.voice.capture", qos: .userInitiated)private let lock = NSLock()private var accepting = true@@ -439,49 +502,69 @@ private final class LiveCaptureDelivery: @unchecked Sendable {private var chunksPending = 0private let stream: DictationStreamprivate let rate: Int- private let interrupted: @Sendable () -> Void+ private let interrupted: @Sendable (Error?) -> Voidprivate let level: @Sendable (Float) -> Voidprivate var meterFrames = 0private var meterEnergy = 0.0init(stream: DictationStream, sampleRate: UInt32,- level: @escaping @Sendable (Float) -> Void, interrupted: @escaping @Sendable () -> Void+ level: @escaping @Sendable (Float) -> Void,+ interrupted: @escaping @Sendable (Error?) -> Void) {- self.stream = stream; rate = Int(sampleRate); self.interrupted = interrupted+ self.stream = stream+ rate = Int(sampleRate)+ self.interrupted = interruptedself.level = level}- func close() { lock.lock(); accepting = false; lock.unlock() }++ func close() {+ lock.lock()+ accepting = false+ lock.unlock()+ }+func drain(_ completion: @escaping @Sendable () -> Void) { queue.async(execute: completion) }func accept(_ buffer: AVAudioPCMBuffer) {let frames = Int(buffer.frameLength)+ accept(frames: frames) {+ guard let channels = buffer.floatChannelData, buffer.format.channelCount > 0 else {+ return nil+ }+ let channelCount = Int(buffer.format.channelCount)+ var mono = [Float](repeating: 0, count: frames)+ for frame in 0..<frames {+ for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }+ mono[frame] /= Float(channelCount)+ }+ return mono+ }+ }++ func accept(frames: Int, copySamples: () -> [Float]?) {guard frames > 0 else { return }lock.lock()- guard accepting else { lock.unlock(); return }+ defer { lock.unlock() }+ guard accepting else { return }guard frames <= rate, framesPending + frames <= rate * 2, chunksPending < 128,- let channels = buffer.floatChannelData, buffer.format.channelCount > 0+ let samples = copySamples()else {accepting = false- lock.unlock()queue.async { [self] in- try? stream.abandon(reason: .failure); interrupted()+ try? stream.abandon(reason: .failure)+ interrupted(nil)}return}framesPending += frameschunksPending += 1- lock.unlock()- let channelCount = Int(buffer.format.channelCount)- var mono = [Float](repeating: 0, count: frames)- for frame in 0..<frames {- for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }- mono[frame] /= Float(channelCount)- }- let samples = monoqueue.async { [self] indefer {- lock.lock(); framesPending -= frames; chunksPending -= 1; lock.unlock()+ lock.lock()+ framesPending -= frames+ chunksPending -= 1+ lock.unlock()}do {meterFrames += frames@@ -489,7 +572,8 @@ private final class LiveCaptureDelivery: @unchecked Sendable {if meterFrames >= rate / 10 {let rms = sqrt(meterEnergy / Double(meterFrames))level(Float(min(1, max(0, (20 * log10(max(rms, 0.00001)) + 55) / 45))))- meterFrames = 0; meterEnergy = 0+ meterFrames = 0+ meterEnergy = 0}for start in stride(from: 0, to: frames, by: rate / 10) {let end = min(frames, start + rate / 10)@@ -509,8 +593,14 @@ private final class LiveCaptureDelivery: @unchecked Sendable {}}} catch {- lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()- if wasAccepting { try? stream.abandon(reason: .failure); interrupted() }+ lock.lock()+ let wasAccepting = accepting+ accepting = false+ lock.unlock()+ if wasAccepting {+ try? stream.abandon(reason: .failure)+ interrupted(error)+ }}}}diff --git a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swiftindex 44292bdc0d..23b45c5b68 100644--- a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift+++ b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift@@ -26,10 +26,6 @@ enum ChatDeviceCapability: Equatable {}}-func supportsLiveVoice(totalMemoryBytes: UInt64 = ProcessInfo.processInfo.physicalMemory) -> Bool {- totalMemoryBytes > 4 * 1024 * 1024 * 1024-}-func currentChatDeviceCapability() -> ChatDeviceCapability {chatDeviceCapability(totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)}diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swiftindex 2765c1b472..cc3ad618bd 100644--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift@@ -20,16 +20,6 @@ private actor ModelOperationBarrier {@MainActorfinal class ModelMemoryCoordinationTests: XCTestCase {- func testLiveVoiceRequiresMoreThanFourGiB() {- let limit: UInt64 = 4 * 1024 * 1024 * 1024- for bytes: UInt64 in [0, 4_000_000_000, limit] {- XCTAssertFalse(supportsLiveVoice(totalMemoryBytes: bytes))- }- for bytes: UInt64 in [limit + 1, 8_000_000_000] {- XCTAssertTrue(supportsLiveVoice(totalMemoryBytes: bytes))- }- }-func testMinimumRamBlocksOnlyKnownReadingsBelowTheFloor() {XCTAssertFalse(chatDeviceCapability(totalMemoryBytes: 1_599_999_999).isChatSupported)XCTAssertTrue(chatDeviceCapability(totalMemoryBytes: 1_600_000_000).isChatSupported)diff --git a/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swift b/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swiftnew file mode 100644index 0000000000..3588c7d767--- /dev/null+++ b/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swift@@ -0,0 +1,56 @@+import Foundation+import XCTest+@testable import Ensu++@MainActor+final class VoiceCaptureDeliveryTests: XCTestCase {+ func testStopDrainsAnAcceptedBufferBeforeClosingTheStream() async throws {+ let owner = DictationSession(live: false)+ let stream = try DictationStream(session: owner, sampleRate: 16000, channels: 1)+ defer { try? stream.abandon(reason: .discard) }+ let copying = expectation(description: "audio copy started")+ let stopping = expectation(description: "stop requested")+ let delivered = expectation(description: "audio delivered")+ let drained = expectation(description: "capture drained")+ let releaseCopy = DispatchSemaphore(value: 0)+ let drainCompleted = DispatchSemaphore(value: 0)+ defer { releaseCopy.signal() }+ let delivery = LiveCaptureDelivery(+ stream: stream, sampleRate: 16000,+ level: { _ in delivered.fulfill() },+ interrupted: { error in+ XCTFail("Unexpected capture failure: \(String(describing: error))")+ }+ )++ DispatchQueue.global().async {+ delivery.accept(frames: 1600) {+ copying.fulfill()+ guard releaseCopy.wait(timeout: .now() + 5) == .success else {+ XCTFail("Audio copy was not released")+ return nil+ }+ return Array(repeating: 0.25, count: 1600)+ }+ }+ await fulfillment(of: [copying], timeout: 2)+ DispatchQueue.global().async {+ stopping.fulfill()+ delivery.close()+ delivery.drain {+ do { try stream.stop(reason: .done) } catch { XCTFail("\(error)") }+ drained.fulfill()+ drainCompleted.signal()+ }+ }+ await fulfillment(of: [stopping], timeout: 2)+ XCTAssertEqual(drainCompleted.wait(timeout: .now() + 0.1), .timedOut)+ releaseCopy.signal()+ await fulfillment(of: [delivered, drained], timeout: 2, enforceOrder: true)+ XCTAssertEqual(try stream.feed(pcmLe: Data([0, 0])), .closed)+ delivery.accept(frames: 1600) {+ XCTFail("Closed capture accepted another buffer")+ return nil+ }+ }+}diff --git a/rust/bindings/uniffi/ensu/src/config.rs b/rust/bindings/uniffi/ensu/src/config.rsindex 3ca8359a02..0c8f35ef94 100644--- a/rust/bindings/uniffi/ensu/src/config.rs+++ b/rust/bindings/uniffi/ensu/src/config.rs@@ -165,6 +165,11 @@ pub fn config_defaults() -> ConfigDefaults {config::defaults().into()}+#[uniffi::export]+pub fn supports_live_voice(total_memory_bytes: Option<u64>) -> bool {+ config::supports_live_voice(total_memory_bytes)+}+#[uniffi::export]pub fn resolve_model_policy(surface: ModelRuntimeSurface,diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rsindex 90683bce04..04e1fb4421 100644--- a/rust/bindings/uniffi/ensu/src/voice.rs+++ b/rust/bindings/uniffi/ensu/src/voice.rs@@ -6,14 +6,37 @@ use std::time::{Duration, Instant};#[derive(Debug, thiserror::Error, uniffi::Error)]pub enum DictationError {+ #[error("Transcription model is not downloaded")]+ NotDownloaded,+ #[error("Voice activity model is not downloaded")]+ VadNotDownloaded,+ #[error("not enough storage space")]+ StorageFull,+ #[error("Voice recording was cancelled")]+ Cancelled,+ #[error("{detail}")]+ InvalidAudio { detail: String },+ #[error("{detail}")]+ Io { detail: String },+ #[error("{detail}")]+ ProcessingFailed { detail: String },#[error("{detail}")]InvalidOperation { detail: String },}impl From<voice::VoiceError> for DictationError {fn from(value: voice::VoiceError) -> Self {- Self::InvalidOperation {- detail: value.to_string(),+ match value {+ voice::VoiceError::NotDownloaded => Self::NotDownloaded,+ voice::VoiceError::VadNotDownloaded => Self::VadNotDownloaded,+ voice::VoiceError::StorageFull => Self::StorageFull,+ voice::VoiceError::Cancelled => Self::Cancelled,+ voice::VoiceError::InvalidAudio(detail) => Self::InvalidAudio { detail },+ voice::VoiceError::Io(detail) => Self::Io { detail },+ voice::VoiceError::ProcessingFailed(detail) => Self::ProcessingFailed { detail },+ error => Self::InvalidOperation {+ detail: error.to_string(),+ },}}}@@ -333,48 +356,6 @@ mod tests {}}- #[test]- fn composer_previews_replace_the_selection_without_consuming_the_transfer() {- let insertion = DictationInsertion::new("chat".into(), 7, 7, 13);- let original = " meet Friday\n".to_owned();- assert_eq!(- insertion- .preview(original.clone(), String::new())- .unwrap()- .text,- original- );- assert_eq!(- insertion- .preview(original.clone(), "Sat".into())- .unwrap()- .text,- " meet Sat\n"- );- let preview = insertion- .preview(original.clone(), "Saturday".into())- .unwrap();- assert_eq!(preview.revision, 7);- let final_text = insertion- .apply("chat".into(), 7, original, "Saturday".into(), true, false)- .unwrap();- assert_eq!(preview.text, final_text.text);- assert_eq!(preview.caret_utf16, final_text.caret_utf16);- assert_eq!(final_text.revision, 8);- assert!(- insertion- .apply(- "chat".into(),- 8,- final_text.text,- "Sunday".into(),- true,- false- )- .is_err()- );- }-fn event(session: &DictationSession, event: LiveEvent) {session.lock().apply(event, || true).unwrap();}@@ -414,18 +395,6 @@ mod tests {assert!(!value.worker_active);assert!(value.incomplete);assert!(matches!(value.phase, DictationPhase::Ready));- let insertion = DictationInsertion::new("chat".into(), 0, 7, 7);- let draft = insertion- .apply(- "chat".into(),- 0,- "Notes: ".into(),- value.text + &value.hypothesis,- true,- false,- )- .unwrap();- assert_eq!(draft.text, "Notes: Keep this opening word");assert!(matches!(owner.lock()@@ -453,14 +422,13 @@ mod tests {let owner = DictationSession::new(live);let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();let signals = Arc::new(StreamSignals::default());- assert!(- stream- .run(- crate::transcription::Transcriber::new("missing".into(), "missing".into()),- signals.clone()- )- .is_err()- );+ assert!(matches!(+ stream.run(+ crate::transcription::Transcriber::new("missing".into(), "missing".into()),+ signals.clone()+ ),+ Err(DictationError::VadNotDownloaded)+ ));assert_eq!(signals.ready.load(Ordering::Relaxed), 0);assert_eq!(signals.stopped.load(Ordering::Relaxed), 1);assert!(!owner.snapshot().worker_active);diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex 197badaa6f..34047a748e 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -206,6 +206,10 @@ pub(crate) fn llm_catalog() -> Vec<ModelPreset> {]}+pub fn supports_live_voice(total_memory_bytes: Option<u64>) -> bool {+ total_memory_bytes.is_some_and(|bytes| bytes > MOBILE_SMALL_MEMORY_MAX_BYTES)+}+pub fn resolve_model_policy(surface: ModelRuntimeSurface,total_memory_bytes: Option<u64>,@@ -370,6 +374,21 @@ mod tests {}}+ #[test]+ fn live_voice_requires_more_than_four_gib() {+ let four_gib = 4 * 1024 * 1024 * 1024;+ for (memory, expected) in [+ (None, false),+ (Some(0), false),+ (Some(four_gib - 1), false),+ (Some(four_gib), false),+ (Some(four_gib + 1), true),+ (Some(8 * 1024 * 1024 * 1024), true),+ ] {+ assert_eq!(supports_live_voice(memory), expected, "RAM: {memory:?}");+ }+ }+#[test]fn mobile_ram_ladder_enforces_inclusive_boundaries_and_saved_preferences() {let gib = 1024 * 1024 * 1024;diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rsindex 0b3d1b9083..faa79cf67c 100644--- a/rust/crates/ensu/src/transcription/audio.rs+++ b/rust/crates/ensu/src/transcription/audio.rs@@ -17,6 +17,7 @@ pub fn extract_speech_from_pcm16(vad_model_path: impl AsRef<Path>,input_sample_rate: u32,pcm_le: &[u8],+ is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {if input_sample_rate == 0 {return Err(TranscriptionError::InvalidAudio(@@ -36,13 +37,14 @@ pub fn extract_speech_from_pcm16(.map(|chunk| i16::from_le_bytes(*chunk) as f32 / 32768.0).collect::<Vec<_>>();- extract_speech(vad_model_path, input_sample_rate, &samples)+ extract_speech(vad_model_path, input_sample_rate, &samples, is_cancelled)}fn extract_speech(vad_model_path: impl AsRef<Path>,input_sample_rate: u32,samples: &[f32],+ is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {if samples.is_empty() {return Ok(Vec::new());@@ -60,13 +62,14 @@ fn extract_speech(VAD_ONSET_FRAMES,);- extract_speech_with_vad(input_sample_rate, samples, &mut vad)+ extract_speech_with_vad(input_sample_rate, samples, &mut vad, is_cancelled)}fn extract_speech_with_vad(input_sample_rate: u32,samples: &[f32],vad: &mut dyn Vad,+ is_cancelled: &mut impl FnMut() -> bool,) -> Result<Vec<f32>> {let mut resampler = FrameResampler::new(input_sample_rate as usize,@@ -88,8 +91,16 @@ fn extract_speech_with_vad(Err(error) => vad_error = Some(error),}};- resampler.push(samples, &mut emit);- resampler.finish(&mut emit);+ for chunk in samples.chunks(input_sample_rate as usize) {+ if is_cancelled() {+ return Ok(Vec::new());+ }+ resampler.push(chunk, &mut emit)?;+ }+ if is_cancelled() {+ return Ok(Vec::new());+ }+ resampler.finish(&mut emit)?;if let Some(error) = vad_error {return Err(error.into());}@@ -143,10 +154,10 @@ impl FrameResampler {})}- fn push(&mut self, mut src: &[f32], mut emit: impl FnMut(&[f32])) {+ fn push(&mut self, mut src: &[f32], mut emit: impl FnMut(&[f32])) -> Result<()> {if self.resampler.is_none() {self.emit_frames(src, &mut emit);- return;+ return Ok(());}while !src.is_empty() {@@ -156,27 +167,38 @@ impl FrameResampler {src = &src[take..];if self.in_buf.len() == self.chunk_in {- if let Some(resampler) = self.resampler.as_mut()- && let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)- && let Ok(out) = resampler.process(&input, None)- {+ if let Some(resampler) = self.resampler.as_mut() {+ let input =+ InterleavedSlice::new(&self.in_buf, 1, self.chunk_in).map_err(|error| {+ TranscriptionError::InvalidAudio(format!(+ "Prepare audio resampler input: {error}"+ ))+ })?;+ let out = resampler.process(&input, None).map_err(|error| {+ TranscriptionError::Transcribe(format!("Resample recorded audio: {error}"))+ })?;self.emit_frames(&out.take_data(), &mut emit);}self.in_buf.clear();}}+ Ok(())}- fn finish(&mut self, mut emit: impl FnMut(&[f32])) {+ fn finish(&mut self, mut emit: impl FnMut(&[f32])) -> Result<()> {if let Some(resampler) = self.resampler.as_mut()&& !self.in_buf.is_empty(){self.in_buf.resize(self.chunk_in, 0.0);- if let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)- && let Ok(out) = resampler.process(&input, None)- {- self.emit_frames(&out.take_data(), &mut emit);- }+ let input = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in).map_err(|error| {+ TranscriptionError::InvalidAudio(format!(+ "Prepare final audio resampler input: {error}"+ ))+ })?;+ let out = resampler.process(&input, None).map_err(|error| {+ TranscriptionError::Transcribe(format!("Resample final recorded audio: {error}"))+ })?;+ self.emit_frames(&out.take_data(), &mut emit);}if !self.pending.is_empty() {@@ -184,6 +206,7 @@ impl FrameResampler {emit(&self.pending);self.pending.clear();}+ Ok(())}fn emit_frames(&mut self, mut data: &[f32], emit: &mut impl FnMut(&[f32])) {@@ -208,9 +231,10 @@ mod tests {use super::*;#[test]- fn vad_failure_does_not_return_an_apparently_complete_partial_recording() {+ fn vad_failure_and_cancellation_do_not_return_partial_recordings() {struct FailsAfterSpeech {frames: usize,+ fail: bool,}impl Vad for FailsAfterSpeech {fn frame_size(&self) -> usize {@@ -221,7 +245,7 @@ mod tests {_: &[f32],) -> std::result::Result<bool, transcribe_rs::TranscribeError> {self.frames += 1;- if self.frames == 1 {+ if self.frames == 1 || !self.fail {Ok(true)} else {Err(transcribe_rs::TranscribeError::Inference(@@ -231,9 +255,27 @@ mod tests {}fn reset(&mut self) {}}- let mut vad = FailsAfterSpeech { frames: 0 };- assert!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad).is_err());+ let mut vad = FailsAfterSpeech {+ frames: 0,+ fail: true,+ };+ assert!(+ matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad, &mut || false), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure"))+ );assert_eq!(vad.frames, 2);+ let mut vad = FailsAfterSpeech {+ frames: 0,+ fail: false,+ };+ let mut checks = 0;+ let speech = extract_speech_with_vad(16000, &vec![0.2; 32000], &mut vad, &mut || {+ checks += 1;+ checks > 1+ })+ .unwrap();+ assert!(speech.is_empty());+ assert_eq!(checks, 2);+ assert_eq!(vad.frames, 16000 / vad.frame_size());}fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {@@ -245,9 +287,9 @@ mod tests {output.extend_from_slice(frame);};for chunk in input.chunks(push_size) {- resampler.push(chunk, &mut emit);+ resampler.push(chunk, &mut emit).unwrap();}- resampler.finish(emit);+ resampler.finish(emit).unwrap();output}diff --git a/rust/crates/ensu/src/transcription/graph_cache.rs b/rust/crates/ensu/src/transcription/graph_cache.rsindex 807f739cc6..2443e98427 100644--- a/rust/crates/ensu/src/transcription/graph_cache.rs+++ b/rust/crates/ensu/src/transcription/graph_cache.rs@@ -546,41 +546,41 @@ mod tests {}#[test]- fn serializer_failure_retries_original_and_cleans_staging() {- let f = Fixture::new();- let result = f.cache.load(&f.source, "runtime-a", true, |plan| {- if matches!(plan, LoadPlan::Source { save_to: Some(_) }) {- f.loads.borrow_mut().push("failed-save");- Err(io::Error::new(- io::ErrorKind::StorageFull,- "injected full disk",- ))- } else {- f.loader(plan)+ fn failed_cache_load_or_serialization_recovers_from_source() {+ for cached in [false, true] {+ let f = Fixture::new();+ let expected = f.loader(LoadPlan::Source { save_to: None }).unwrap();+ if cached {+ assert_eq!(f.load("runtime-a"), expected);}- });- assert!(result.is_ok());- assert_eq!(f.loads.take(), ["failed-save", "source"]);- assert!(!f.ready().exists());- assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 1);- }-- #[test]- fn rejected_hash_valid_graph_rebuilds_from_source() {- let f = Fixture::new();- let expected = f.load("runtime-a");- let actual = f- .cache- .load(&f.source, "runtime-a", true, |plan| {- if matches!(plan, LoadPlan::Cached { .. }) {- Err(invalid_cache())- } else {- f.loader(plan)- }- })- .unwrap();- assert_eq!(actual, expected);- assert_eq!(f.loads.take(), ["build", "build"]);+ f.loads.take();+ let mut failed = false;+ let actual = f+ .cache+ .load(&f.source, "runtime-a", true, |plan| match plan {+ LoadPlan::Cached { .. } if cached => {+ failed = true;+ Err(invalid_cache())+ }+ LoadPlan::Source { save_to: Some(_) } if !cached => {+ failed = true;+ Err(io::Error::new(+ io::ErrorKind::StorageFull,+ "injected full disk",+ ))+ }+ _ => f.loader(plan),+ })+ .unwrap();+ assert!(failed);+ assert_eq!(actual, expected);+ assert_eq!(f.loads.take(), [if cached { "build" } else { "source" }]);+ assert_eq!(f.ready().exists(), cached);+ assert_eq!(+ fs::read_dir(&f.cache.root).unwrap().count(),+ if cached { 2 } else { 1 }+ );+ }}#[test]diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rsindex 7c281996dc..9d838e9933 100644--- a/rust/crates/ensu/src/transcription/live.rs+++ b/rust/crates/ensu/src/transcription/live.rs@@ -119,14 +119,10 @@ impl LiveTranscription {|| {transcriber.load_model()- .map_err(|_| VoiceError::ProcessingFailed)+ .map_err(|error| super::voice_error("Load transcription model", error))},|request| transcriber.transcribe_window(&request.samples),- |pcm| {- transcriber- .transcribe(16_000, pcm)- .map_err(|_| VoiceError::ProcessingFailed)- },+ |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),on_event,)}@@ -142,7 +138,9 @@ impl LiveTranscription {let _capture = self.start()?;let mut vad = make_vad()?;if vad.frame_size() != super::VAD_FRAME_SAMPLES {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Voice activity model has an unsupported frame size".into(),+ ));}let mut normalizer = StreamNormalizer::new(self.queue.format())?;let mut rolling = RollingRecognizer::default();@@ -183,9 +181,9 @@ impl LiveTranscription {let chunk = self.queue.take();let had_chunk = chunk.is_some();let mut emit = |frame: &[f32], real: usize| -> Result<()> {- let voiced = vad- .is_speech(frame)- .map_err(|_| VoiceError::ProcessingFailed)?;+ let voiced = vad.is_speech(frame).map_err(|error| {+ super::voice_error("Detect speech in live audio", error)+ })?;let start = accepted;accepted += real as u64;self.emit(@@ -262,8 +260,10 @@ impl LiveTranscription {}}Err(TryRecvError::Empty) => {}- Err(TryRecvError::Disconnected) => {- return Err(VoiceError::ProcessingFailed);+ Err(error @ TryRecvError::Disconnected) => {+ return Err(VoiceError::ProcessingFailed(format!(+ "Receive speech decoder result: {error}"+ )));}}if prepared && finished && (deferred || rolling.is_settled()) {@@ -276,9 +276,11 @@ impl LiveTranscription {&& !deferred&& let Some(request) = rolling.next_window()?{- requests- .try_send(request)- .map_err(|_| VoiceError::ProcessingFailed)?;+ requests.try_send(request).map_err(|error| {+ VoiceError::ProcessingFailed(format!(+ "Send speech decoder window: {error}"+ ))+ })?;}if !had_chunk && !finished {self.emit(LiveEvent::Idle, on_event)?;@@ -294,7 +296,9 @@ impl LiveTranscription {}drop(requests);drop(result_rx);- let exited = decoder.join().map_err(|_| VoiceError::ProcessingFailed);+ let exited = decoder+ .join()+ .map_err(|_| VoiceError::ProcessingFailed("Speech decoder thread panicked".into()));exited.and(processing)?;if deferred {drop(vad);@@ -492,25 +496,55 @@ mod tests {}#[test]- fn failed_model_initialization_closes_capture_without_recognition() {- let stream = stream();- let mut closed = false;- let result = stream.run_with(- vad,- || Err(VoiceError::ProcessingFailed),- |_| panic!("Failed model must not decode"),- |_| panic!("Failed model must not transcribe"),- &mut |event| {- match event {- LiveEvent::Ready | LiveEvent::Idle => {}- LiveEvent::CaptureClosed => closed = true,- _ => panic!("Failed initialization emitted recognition"),- }- Ok(LiveControl::Continue)- },- );- assert_eq!(result, Err(VoiceError::ProcessingFailed));- assert!(closed);+ fn model_initialization_failures_preserve_causes_and_close_capture() {+ use super::super::TranscriptionError;+ use transcribe_rs::TranscribeError;++ for (error, expected) in [+ (TranscriptionError::NotDownloaded, VoiceError::NotDownloaded),+ (+ TranscriptionError::VadNotDownloaded,+ VoiceError::VadNotDownloaded,+ ),+ (+ TranscribeError::Io(std::io::ErrorKind::StorageFull.into()).into(),+ VoiceError::StorageFull,+ ),+ (+ TranscribeError::Io(std::io::Error::other("injected I/O failure")).into(),+ VoiceError::Io("Load transcription model: injected I/O failure".into()),+ ),+ (+ TranscribeError::Audio("injected audio failure".into()).into(),+ VoiceError::InvalidAudio("Load transcription model: injected audio failure".into()),+ ),+ (+ TranscribeError::Inference("injected initialization failure".into()).into(),+ VoiceError::ProcessingFailed(+ "Load transcription model: inference error: injected initialization failure"+ .into(),+ ),+ ),+ ] {+ let stream = stream();+ let mut closed = false;+ let result = stream.run_with(+ vad,+ || Err(super::super::voice_error("Load transcription model", error)),+ |_| panic!("Failed model must not decode"),+ |_| panic!("Failed model must not transcribe"),+ &mut |event| {+ match event {+ LiveEvent::Ready | LiveEvent::Idle => {}+ LiveEvent::CaptureClosed => closed = true,+ _ => panic!("Failed initialization emitted recognition"),+ }+ Ok(LiveControl::Continue)+ },+ );+ assert_eq!(result, Err(expected));+ assert!(closed);+ }}#[test]diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rsindex 651de4da09..37720cdb56 100644--- a/rust/crates/ensu/src/transcription/mod.rs+++ b/rust/crates/ensu/src/transcription/mod.rs@@ -41,7 +41,12 @@ impl From<std::io::Error> for TranscriptionError {impl From<transcribe_rs::TranscribeError> for TranscriptionError {fn from(value: transcribe_rs::TranscribeError) -> Self {- Self::Transcribe(value.to_string())+ match value {+ transcribe_rs::TranscribeError::ModelNotFound(_) => Self::NotDownloaded,+ transcribe_rs::TranscribeError::Audio(detail) => Self::InvalidAudio(detail),+ transcribe_rs::TranscribeError::Io(error) => error.into(),+ error => Self::Transcribe(error.to_string()),+ }}}@@ -57,4 +62,20 @@ impl From<rubato::ResamplerConstructionError> for TranscriptionError {}}+fn voice_error(operation: &str, error: impl Into<TranscriptionError>) -> crate::voice::VoiceError {+ use crate::voice::VoiceError;+ match error.into() {+ TranscriptionError::NotDownloaded => VoiceError::NotDownloaded,+ TranscriptionError::VadNotDownloaded => VoiceError::VadNotDownloaded,+ TranscriptionError::StorageFull => VoiceError::StorageFull,+ TranscriptionError::InvalidAudio(detail) => {+ VoiceError::InvalidAudio(format!("{operation}: {detail}"))+ }+ TranscriptionError::Transcribe(detail) => {+ VoiceError::ProcessingFailed(format!("{operation}: {detail}"))+ }+ TranscriptionError::Io(error) => VoiceError::Io(format!("{operation}: {error}")),+ }+}+pub type Result<T> = std::result::Result<T, TranscriptionError>;diff --git a/rust/crates/ensu/src/transcription/normalize.rs b/rust/crates/ensu/src/transcription/normalize.rsindex 66e50b9d98..bfc0651868 100644--- a/rust/crates/ensu/src/transcription/normalize.rs+++ b/rust/crates/ensu/src/transcription/normalize.rs@@ -32,7 +32,7 @@ impl StreamNormalizer {WindowFunction::BlackmanHarris2,FixedSync::Input,)- .map_err(|_| VoiceError::InvalidAudio)?,+ .map_err(|error| super::voice_error("Create audio resampler", error))?,)};let delay = resampler.as_ref().map_or(0, Resampler::output_delay);@@ -55,7 +55,9 @@ impl StreamNormalizer {) -> Result<()> {let frame_bytes = self.format.bytes_per_frame();if self.finished || !pcm.len().is_multiple_of(frame_bytes) {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Audio must contain complete PCM16 frames before normalization finishes".into(),+ ));}for frame in pcm.chunks_exact(frame_bytes) {let mono = frame@@ -89,7 +91,9 @@ impl StreamNormalizer {let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);while self.emitted < expected {let Some(resampler) = &self.resampler else {- return Err(VoiceError::ProcessingFailed);+ return Err(VoiceError::ProcessingFailed(+ "Audio normalization did not produce all input samples".into(),+ ));};self.input.resize(resampler.input_frames_next(), 0.0);self.process_chunk(emit)?;@@ -106,11 +110,14 @@ impl StreamNormalizer {fn process_chunk(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {let resampler = self.resampler.as_mut().ok_or(VoiceError::InvalidState)?;- let input = InterleavedSlice::new(&self.input, 1, self.input.len())- .map_err(|_| VoiceError::InvalidAudio)?;+ let input = InterleavedSlice::new(&self.input, 1, self.input.len()).map_err(|error| {+ VoiceError::InvalidAudio(format!("Prepare audio resampler input: {error}"))+ })?;let output = resampler.process(&input, None)- .map_err(|_| VoiceError::ProcessingFailed)?+ .map_err(|error| {+ VoiceError::ProcessingFailed(format!("Resample captured audio: {error}"))+ })?.take_data();self.input.clear();let skip = self.delay.min(output.len());diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rsindex 69f7c3f243..133eeaf934 100644--- a/rust/crates/ensu/src/transcription/recorded.rs+++ b/rust/crates/ensu/src/transcription/recorded.rs@@ -7,6 +7,45 @@ use crate::voice::{Result, VoiceError};const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;+pub(super) fn transcribe_chunks(+ mut samples: &[f32],+ mut decode: impl FnMut(&[f32]) -> super::Result<Option<String>>,+) -> super::Result<Option<String>> {+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE, transcript_separator};++ let mut text = String::new();+ while !samples.is_empty() {+ let end = if samples.len() <= MAX_WINDOW_SAMPLES {+ samples.len()+ } else {+ let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);+ let start = limit - 2 * VOICE_SAMPLE_RATE as usize;+ let offset = samples[start..limit]+ .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()+ .0+ .iter()+ .enumerate()+ .map(|(index, frame)| {+ (+ index,+ frame.iter().map(|sample| sample * sample).sum::<f32>(),+ )+ })+ .min_by(|a, b| a.1.total_cmp(&b.1))+ .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);+ start + offset+ };+ let Some(chunk) = decode(&samples[..end])? else {+ return Ok(None);+ };+ let chunk = chunk.trim();+ text.push_str(transcript_separator(&text, chunk));+ text.push_str(chunk);+ samples = &samples[end..];+ }+ Ok(Some(text))+}+#[derive(Default)]pub(super) struct RecordedPcm(Vec<u8>);@@ -41,11 +80,7 @@ impl LiveTranscription {) -> Result<()> {self.run_recorded_with(|| transcriber.live_vad(),- |pcm| {- transcriber- .transcribe(16_000, pcm)- .map_err(|_| VoiceError::ProcessingFailed)- },+ |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),on_event,)}@@ -69,9 +104,9 @@ impl LiveTranscription {let chunk = self.queue.take();let had_chunk = chunk.is_some();let mut emit = |frame: &[f32], real: usize| -> Result<()> {- let voiced = vad- .is_speech(frame)- .map_err(|_| VoiceError::ProcessingFailed)?;+ let voiced = vad.is_speech(frame).map_err(|error| {+ super::voice_error("Detect speech in recorded audio", error)+ })?;heard_speech |= voiced;let start = accepted;accepted += real as u64;@@ -125,6 +160,42 @@ mod tests {use crate::voice::CaptureFormat;use transcribe_rs::vad::EnergyVad;+ #[test]+ fn recorded_decoding_bounds_windows_preserves_tails_and_stops_on_cancel() {+ use crate::voice::MAX_WINDOW_SAMPLES;++ let mut samples = vec![0.25; MAX_WINDOW_SAMPLES * 3 + 1];+ let quiet = MAX_WINDOW_SAMPLES - 2 * 16000 + 10 * super::super::VAD_FRAME_SAMPLES;+ samples[quiet..quiet + super::super::VAD_FRAME_SAMPLES].fill(0.0);+ let mut offset = 0;+ let mut decoded = 0;+ let text = transcribe_chunks(&samples, |chunk| {+ assert!(chunk.len() <= MAX_WINDOW_SAMPLES);+ assert!(chunk.len() >= 16000);+ assert_eq!(chunk, &samples[offset..offset + chunk.len()]);+ if offset == 0 {+ assert_eq!(chunk.len(), quiet + super::super::VAD_FRAME_SAMPLES);+ }+ offset += chunk.len();+ decoded += 1;+ Ok(Some(" again ".into()))+ })+ .unwrap()+ .unwrap();+ assert_eq!(offset, samples.len());+ assert_eq!(text.split_whitespace().count(), decoded);+ let mut calls = 0;+ assert!(+ transcribe_chunks(&samples, |_| {+ calls += 1;+ Ok((calls == 1).then(|| "unfinished".into()))+ })+ .unwrap()+ .is_none()+ );+ assert_eq!(calls, 2);+ }+#[test]fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);@@ -138,19 +209,19 @@ mod tests {}#[test]- fn recorded_text_arrives_once_after_close_with_partial_tail() {- let stream = LiveTranscription::new(CaptureFormat {- sample_rate: 16000,- channels: 1,- })- .unwrap();- stream.feed(&[0, 32].repeat(1001)).unwrap();- stream.finish();- let mut closed = false;- let mut commits = Vec::new();- let mut covered = 0;- stream- .run_recorded_with(+ fn recorded_completion_preserves_the_tail_and_cancellation_suppresses_results() {+ for cancelled in [false, true] {+ let stream = LiveTranscription::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap();+ stream.feed(&[0, 32].repeat(1001)).unwrap();+ stream.finish();+ let mut closed = 0;+ let mut commits = Vec::new();+ let mut covered = None;+ let result = stream.run_recorded_with(|| {Ok(Box::new(EnergyVad::new(super::super::VAD_FRAME_SAMPLES,@@ -158,27 +229,37 @@ mod tests {)))},|pcm| {- assert_eq!(pcm.len(), 1001 * 2);assert_eq!(pcm, [0, 32].repeat(1001));+ if cancelled {+ stream.cancel();+ }Ok("dictated words".into())},&mut |event| {match event {- LiveEvent::CaptureClosed => closed = true,+ LiveEvent::CaptureClosed => closed += 1,LiveEvent::ReplaceTranscript(text) => {- assert!(closed);+ assert_eq!(closed, 1);commits.push(text);}LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),- LiveEvent::Coverage(end) => covered = end,+ LiveEvent::Coverage(end) => covered = Some(end),_ => {}}Ok(LiveControl::Continue)},- )- .unwrap();- assert_eq!(commits, ["dictated words"]);- assert_eq!(covered, 1001);+ );+ assert_eq!(closed, 1);+ if cancelled {+ assert_eq!(result, Err(VoiceError::Cancelled));+ assert!(commits.is_empty());+ assert_eq!(covered, None);+ } else {+ result.unwrap();+ assert_eq!(commits, ["dictated words"]);+ assert_eq!(covered, Some(1001));+ }+ }}#[test]@@ -217,7 +298,6 @@ mod tests {}let control = state.apply(event, || stream.finish_if_empty())?;if idle && state.session.stop_reason() == Some(StopReason::Silence) {- // A native read can finish after closure, before its stop callback.assert!(!capture_closed);assert_eq!(stream.feed(&[0; 3200])?, CaptureFeedResult::Closed);late_buffer = true;@@ -236,35 +316,4 @@ mod tests {assert_eq!(state.text(), "Complete recorded transcript");assert_eq!(stream.submitted_samples(), (audio.len() / 2) as u64);}-- #[test]- fn cancellation_during_batch_decode_does_not_publish_text() {- let stream = LiveTranscription::new(CaptureFormat {- sample_rate: 16000,- channels: 1,- })- .unwrap();- stream.feed(&[0, 32].repeat(512)).unwrap();- stream.finish();- let result = stream.run_recorded_with(- || {- Ok(Box::new(EnergyVad::new(- super::super::VAD_FRAME_SAMPLES,- 0.01,- )))- },- |_| {- stream.cancel();- Ok("late result".into())- },- &mut |event| {- assert!(!matches!(- event,- LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)- ));- Ok(LiveControl::Continue)- },- );- assert_eq!(result, Err(VoiceError::Cancelled));- }}diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rsindex 8c88eda0db..cee8f3ae32 100644--- a/rust/crates/ensu/src/transcription/session.rs+++ b/rust/crates/ensu/src/transcription/session.rs@@ -59,7 +59,9 @@ impl DictationState {}LiveEvent::Commit { start, end, text } => {if end <= start || start < self.last_commit_end {- return Err(voice::VoiceError::InvalidAudio);+ return Err(voice::VoiceError::InvalidAudio(+ "Transcript commits must have non-overlapping audio ranges".into(),+ ));}self.last_commit_end = end;self.session.observe_recognition(!text.trim().is_empty())?;@@ -149,21 +151,6 @@ mod tests {);}- #[test]- fn preview_preserves_native_spaces_and_unspaced_scripts() {- for (stable, partial, expected) in [- ("ζ", "倩", "ζ倩"),- ("hello", "world", "hello world"),- ("hello ", "world", "hello world"),- ] {- let mut state = DictationState::new(true);- state.session.begin_capture().unwrap();- event(&mut state, LiveEvent::ReplaceTranscript(stable.into()));- event(&mut state, LiveEvent::Hypothesis(partial.into()));- assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);- }- }-#[test]fn recognized_word_boundaries_survive_preview_and_commit() {fn apply_update(state: &mut DictationState, update: voice::RecognitionUpdate) {@@ -175,6 +162,15 @@ mod tests {}for (words, expected) in [+ (vec!["ζ", "倩", "θ§", "ι’"], "ζ倩θ§ι’"),+ (+ vec!["hello", "world", "again", "today"],+ "hello world again today",+ ),+ (+ vec!["hello", "world ", "again", "today"],+ "hello world again today",+ ),(vec!["It", "costs", "$20.", "today."],"It costs $20. today.",diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rsindex 16a34ba8ee..a23c23afed 100644--- a/rust/crates/ensu/src/transcription/transcriber.rs+++ b/rust/crates/ensu/src/transcription/transcriber.rs@@ -18,9 +18,19 @@ pub struct Transcriber {impl Transcriber {pub(crate) fn live_vad(&self) -> crate::voice::Result<Box<dyn transcribe_rs::vad::Vad>> {+ let metadata = std::fs::metadata(&self.vad_model_path).map_err(|error| {+ if error.kind() == std::io::ErrorKind::NotFound {+ crate::voice::VoiceError::VadNotDownloaded+ } else {+ super::voice_error("Read voice activity model", error)+ }+ })?;+ if !metadata.is_file() {+ return Err(crate::voice::VoiceError::VadNotDownloaded);+ }super::SileroV4Vad::new(&self.vad_model_path).map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>)- .map_err(|_| crate::voice::VoiceError::ProcessingFailed)+ .map_err(|error| super::voice_error("Load voice activity model", error))}pub(crate) fn transcribe_window(@@ -32,7 +42,7 @@ impl Transcriber {|| samples.len() > MAX_WINDOW_SAMPLES|| samples.iter().any(|s| !s.is_finite()){- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio("Recognition requires a nonempty window of finite samples within the duration limit".into()));}let mut loaded = self.lock();let model = ensure_loaded(@@ -41,7 +51,7 @@ impl Transcriber {self.graph_cache.as_ref(),true,)- .map_err(|_| VoiceError::ProcessingFailed)?;+ .map_err(|error| super::voice_error("Load transcription model", error))?;let result = model.transcribe_with(samples,@@ -50,7 +60,7 @@ impl Transcriber {..Default::default()},)- .map_err(|_| VoiceError::ProcessingFailed)?;+ .map_err(|error| super::voice_error("Transcribe live audio", error))?;result.segments.ok_or(VoiceError::UnstableTranscript)?@@ -110,8 +120,31 @@ impl Transcriber {}pub fn transcribe(&self, input_sample_rate: u32, pcm_le: &[u8]) -> Result<String> {+ self.transcribe_with_cancel(input_sample_rate, pcm_le, || false)+ .map(Option::unwrap_or_default)+ }++ pub(crate) fn transcribe_recording(+ &self,+ pcm_le: &[u8],+ is_cancelled: impl FnMut() -> bool,+ ) -> crate::voice::Result<String> {+ self.transcribe_with_cancel(16_000, pcm_le, is_cancelled)+ .map_err(|error| super::voice_error("Transcribe recorded audio", error))?+ .ok_or(crate::voice::VoiceError::Cancelled)+ }++ fn transcribe_with_cancel(+ &self,+ input_sample_rate: u32,+ pcm_le: &[u8],+ mut is_cancelled: impl FnMut() -> bool,+ ) -> Result<Option<String>> {+ if is_cancelled() {+ return Ok(None);+ }if pcm_le.is_empty() {- return Ok(String::new());+ return Ok(Some(String::new()));}if !self.model_dir.is_dir() {return Err(TranscriptionError::NotDownloaded);@@ -120,9 +153,17 @@ impl Transcriber {return Err(TranscriptionError::VadNotDownloaded);}- let speech = extract_speech_from_pcm16(&self.vad_model_path, input_sample_rate, pcm_le)?;+ let speech = extract_speech_from_pcm16(+ &self.vad_model_path,+ input_sample_rate,+ pcm_le,+ &mut is_cancelled,+ )?;+ if is_cancelled() {+ return Ok(None);+ }if speech.is_empty() {- return Ok(String::new());+ return Ok(Some(String::new()));}let mut loaded = self.lock();@@ -132,15 +173,19 @@ impl Transcriber {self.graph_cache.as_ref(),false,)?;- let result = model.transcribe_with(- &speech,- &ParakeetParams {- timestamp_granularity: Some(TimestampGranularity::Segment),- ..Default::default()- },- )?;-- Ok(result.text.trim().to_owned())+ super::recorded::transcribe_chunks(&speech, |samples| {+ if is_cancelled() {+ return Ok(None);+ }+ let result = model.transcribe_with(+ samples,+ &ParakeetParams {+ timestamp_granularity: Some(TimestampGranularity::Segment),+ ..Default::default()+ },+ )?;+ Ok((!is_cancelled()).then_some(result.text))+ })}fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {diff --git a/rust/crates/ensu/src/voice/audio.rs b/rust/crates/ensu/src/voice/audio.rsindex 5ee6a89eb2..9583cf9ad5 100644--- a/rust/crates/ensu/src/voice/audio.rs+++ b/rust/crates/ensu/src/voice/audio.rs@@ -2,13 +2,13 @@ use std::collections::VecDeque;use super::{Result, VoiceError};-pub const VOICE_SAMPLE_RATE: u64 = 16_000;-pub const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;-pub const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;-pub const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;+pub(crate) const VOICE_SAMPLE_RATE: u64 = 16_000;+pub(crate) const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;+const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;+const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;#[derive(Debug)]-pub struct AudioRing {+pub(crate) struct AudioRing {samples: VecDeque<f32>,start: u64,processed: u64,@@ -27,11 +27,13 @@ impl Default for AudioRing {impl AudioRing {pub fn push(&mut self, offset: u64, samples: &[f32]) -> Result<()> {if offset != self.accepted() || samples.iter().any(|v| !v.is_finite()) {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Invalid audio offset or samples".into(),+ ));}let end = offset.checked_add(samples.len() as u64)- .ok_or(VoiceError::InvalidAudio)?;+ .ok_or_else(|| VoiceError::InvalidAudio("Audio offset overflow".into()))?;if end - self.processed > MAX_BACKLOG_SAMPLES|| self.samples.len().saturating_add(samples.len()) > MAX_RING_SAMPLES{@@ -47,7 +49,9 @@ impl AudioRing {|| end < start|| end - start > MAX_WINDOW_SAMPLES as u64{- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Audio window is outside the retained range".into(),+ ));}Ok(self.samples@@ -64,7 +68,9 @@ impl AudioRing {|| retain_from < self.start|| retain_from > processed{- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Invalid processed audio range".into(),+ ));}self.samples.drain(..(retain_from - self.start) as usize);self.start = retain_from;@@ -112,10 +118,19 @@ mod tests {#[test]fn invalid_offsets_and_nan_are_rejected_atomically() {let mut ring = AudioRing::default();- assert_eq!(ring.push(1, &[0.0]), Err(VoiceError::InvalidAudio));- assert_eq!(ring.push(0, &[f32::NAN]), Err(VoiceError::InvalidAudio));+ assert!(matches!(+ ring.push(1, &[0.0]),+ Err(VoiceError::InvalidAudio(_))+ ));+ assert!(matches!(+ ring.push(0, &[f32::NAN]),+ Err(VoiceError::InvalidAudio(_))+ ));assert_eq!(ring.accepted(), 0);ring.push(0, &[0.1, 0.2]).unwrap();- assert_eq!(ring.acknowledge(3, 0), Err(VoiceError::InvalidAudio));+ assert!(matches!(+ ring.acknowledge(3, 0),+ Err(VoiceError::InvalidAudio(_))+ ));}}diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rsindex 675782eb50..569ad6340b 100644--- a/rust/crates/ensu/src/voice/capture.rs+++ b/rust/crates/ensu/src/voice/capture.rs@@ -19,12 +19,14 @@ pub struct CaptureFormat {impl CaptureFormat {pub fn validate(self) -> Result<Self> {if !(8_000..=48_000).contains(&self.sample_rate) || !matches!(self.channels, 1 | 2) {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Capture requires 8β48 kHz PCM16 with one or two channels".into(),+ ));}Ok(self)}- pub fn bytes_per_frame(self) -> usize {+ pub(crate) fn bytes_per_frame(self) -> usize {usize::from(self.channels) * 2}}@@ -36,7 +38,7 @@ struct Pending {closed: bool,}-pub struct CaptureQueue {+pub(crate) struct CaptureQueue {format: CaptureFormat,pending: Mutex<Pending>,failed: AtomicBool,@@ -72,7 +74,10 @@ impl CaptureQueue {|| pcm.len() / frame_bytes > self.format.sample_rate as usize / 5{self.failed.store(true, Ordering::Release);- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Capture buffer must contain complete PCM16 frames and at most 200 ms of audio"+ .into(),+ ));}let owned = pcm.to_vec();let mut pending = self.lock();diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rsindex 8f22dadbf5..8357ba0995 100644--- a/rust/crates/ensu/src/voice/composer.rs+++ b/rust/crates/ensu/src/voice/composer.rs@@ -2,7 +2,7 @@ use std::ops::Range;use super::{Result, VoiceError};-pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {+fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {if start > end {return Err(VoiceError::InvalidTextRange);}@@ -34,8 +34,6 @@ fn left_edge(text: &str) -> Option<char> {.or_else(|| text.chars().next())}-// Recognition already supplies word boundaries. Symbols and punctuation belong to-// those words and must not suppress their separators, including across commits.pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {fn unspaced(c: char) -> bool {matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)@@ -50,7 +48,7 @@ pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {}}-pub fn boundary_separator(left: &str, right: &str) -> &'static str {+fn boundary_separator(left: &str, right: &str) -> &'static str {fn opening(c: char) -> bool {matches!(c, 'ΒΏ' | 'Β‘')|| regex::regex!(r"\A[\p{Ps}\p{Pi}]\z").is_match(c.encode_utf8(&mut [0; 4]))@@ -103,8 +101,6 @@ pub fn boundary_separator(left: &str, right: &str) -> &'static str {return "";}- // A currency prefix attaches to its amount; a trailing currency symbol is- // already part of a completed amount and needs a word separator.if regex::regex!(r"\A\p{Sc}\z").is_match(a.encode_utf8(&mut [0; 4]))&& !left.rsplit_once(a).is_some_and(|(before, _)| {before@@ -173,7 +169,7 @@ pub fn preview_insertion(#[derive(Debug, Default)]pub struct ComposerTransfer {- receipt: Option<(VoiceInsertion, u32)>,+ applied: bool,}impl ComposerTransfer {@@ -186,12 +182,8 @@ impl ComposerTransfer {foreground: bool,composing: bool,) -> Result<u32> {- if let Some((applied, receipt)) = &self.receipt {- return if applied == insertion {- Ok(*receipt)- } else {- Err(VoiceError::RevisionConflict)- };+ if self.applied {+ return Err(VoiceError::RevisionConflict);}if !foreground || insertion.anchor.destination != destination {return Err(VoiceError::RevisionConflict);@@ -210,7 +202,7 @@ impl ComposerTransfer {let (rendered, caret) = preview_insertion(text, &anchor, &insertion.text)?;*text = rendered;*revision += 1;- self.receipt = Some((insertion.clone(), caret));+ self.applied = true;Ok(caret)}}@@ -232,27 +224,25 @@ mod tests {}#[test]- fn selected_text_is_replaced_once_even_after_lost_acknowledgement() {+ fn selected_text_is_replaced_once_and_duplicate_transfers_are_rejected() {let mut owner = ComposerTransfer::default();let mut text = " meet Friday\n".to_owned();let mut revision = 7;let insert = insertion(7, 13, "Saturday");- let receipt = owner+ owner.apply(&insert, "chat", &mut revision, &mut text, true, false).unwrap();assert_eq!(text, " meet Saturday\n");assert_eq!(- owner- .apply(- &insert,- "different-chat",- &mut revision,- &mut text,- false,- false- )- .unwrap(),- receipt+ owner.apply(+ &insert,+ "different-chat",+ &mut revision,+ &mut text,+ false,+ false+ ),+ Err(VoiceError::RevisionConflict));assert_eq!(revision, 8);assert_eq!(text, " meet Saturday\n");@@ -287,9 +277,10 @@ mod tests {.apply(&insert, "chat", &mut revision, &mut text, true, false).unwrap();assert_eq!(text, "My edited draft hello");- owner- .apply(&insert, "chat", &mut revision, &mut text, true, false)- .unwrap();+ assert_eq!(+ owner.apply(&insert, "chat", &mut revision, &mut text, true, false),+ Err(VoiceError::RevisionConflict)+ );assert_eq!(text, "My edited draft hello");assert_eq!(revision, 9);}@@ -379,7 +370,6 @@ mod tests {caret);assert_eq!(text, expected);- // The same boundary can occur after the inserted text, too.let anchor = insertion(0, 0, draft).anchor;assert_eq!(preview_insertion(dictated, &anchor, draft).unwrap().0,diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rsindex dc70c90c0c..2d466e5a66 100644--- a/rust/crates/ensu/src/voice/mod.rs+++ b/rust/crates/ensu/src/voice/mod.rs@@ -4,24 +4,36 @@ mod composer;mod rolling;mod session;-pub use audio::*;-pub use capture::*;-pub use composer::*;-pub use rolling::*;-pub use session::*;+pub(crate) use audio::{AudioRing, MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};+pub(crate) use capture::CaptureQueue;+pub use capture::{CaptureFeedResult, CaptureFormat};+pub(crate) use composer::transcript_separator;+pub use composer::{ComposerAnchor, ComposerTransfer, VoiceInsertion, preview_insertion};+#[cfg(test)]+pub(crate) use rolling::RecognitionUpdate;+pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer};+pub use session::{StopReason, VoicePhase, VoiceSession};#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]pub enum VoiceError {- #[error("Audio continuity or format is invalid")]- InvalidAudio,+ #[error("{0}")]+ InvalidAudio(String),+ #[error("Transcription model is not downloaded")]+ NotDownloaded,+ #[error("Voice activity model is not downloaded")]+ VadNotDownloaded,+ #[error("not enough storage space")]+ StorageFull,+ #[error("{0}")]+ Io(String),#[error("Audio backlog limit reached")]BacklogFull,#[error("Voice recording was cancelled")]Cancelled,#[error("Speech alignment could not be preserved")]UnstableTranscript,- #[error("Speech processing failed")]- ProcessingFailed,+ #[error("{0}")]+ ProcessingFailed(String),#[error("Voice event belongs to an inactive recording")]StaleEvent,#[error("Voice operation is not valid in the current state")]diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rsindex cbd9e9050e..25d06ae8bc 100644--- a/rust/crates/ensu/src/voice/rolling.rs+++ b/rust/crates/ensu/src/voice/rolling.rs@@ -13,13 +13,13 @@ const REQUIRED_OBSERVATIONS: u8 = 3;const RIGHT_CONTEXT_WORDS: usize = 2;#[derive(Clone, Debug)]-pub struct AlignedWord {+pub(crate) struct AlignedWord {pub text: String,pub start: u64,pub end: u64,}-pub struct DecodeWindow {+pub(crate) struct DecodeWindow {pub id: u64,pub start: u64,pub end: u64,@@ -28,7 +28,7 @@ pub struct DecodeWindow {}#[derive(Debug)]-pub struct RecognitionUpdate {+pub(crate) struct RecognitionUpdate {pub commit: Option<(u64, u64, String)>,pub hypothesis: String,pub covered_through: u64,@@ -41,7 +41,7 @@ struct Region {}#[derive(Default)]-pub struct RollingRecognizer {+pub(crate) struct RollingRecognizer {ring: AudioRing,regions: VecDeque<Region>,previous: Vec<AlignedWord>,@@ -56,7 +56,7 @@ pub struct RollingRecognizer {impl RollingRecognizer {pub fn feed(&mut self, samples: &[f32], voiced: bool) -> Result<()> {if samples.is_empty() {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio("Audio buffer is empty".into()));}let start = self.ring.accepted();self.ring.push(start, samples)?;diff --git a/rust/crates/ensu/src/voice/session.rs b/rust/crates/ensu/src/voice/session.rsindex 613015d145..a15b3a9024 100644--- a/rust/crates/ensu/src/voice/session.rs+++ b/rust/crates/ensu/src/voice/session.rs@@ -103,7 +103,9 @@ impl VoiceSession {|| end <= start|| voiced_end.is_some_and(|v| v <= start || v > end){- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Audio must be continuous and contain a valid speech range".into(),+ ));}self.accepted = end;if let Some(voice) = voiced_end {@@ -182,7 +184,9 @@ impl VoiceSession {return Err(VoiceError::InvalidState);}if end < self.recognized || end > self.accepted {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Recognition coverage is outside the accepted audio range".into(),+ ));}self.recognized = end;Ok(())@@ -192,7 +196,9 @@ impl VoiceSession {return Err(VoiceError::InvalidState);}if start != self.accepted || end <= start {- return Err(VoiceError::InvalidAudio);+ return Err(VoiceError::InvalidAudio(+ "Drained audio must continue the accepted audio range".into(),+ ));}self.accepted = end;Ok(())@@ -255,38 +261,40 @@ mod tests {}#[test]- fn live_grace_waits_for_text_or_fifteen_seconds() {- let mut session = VoiceSession::default();- session.begin_capture().unwrap();- session.observe_audio(0, 32000, Some(24000)).unwrap();- session.observe_recognition(false).unwrap();- session.observe_audio(32000, 80000, None).unwrap();- assert_eq!(session.phase(), VoicePhase::Listening);- session.observe_audio(80000, 13 * 16000, None).unwrap();- assert!(!session.silence_elapsed());- assert_eq!(session.silence_deadline(), Some(15 * 16000));- session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();- assert!(session.silence_elapsed());- }-- #[test]- fn recognition_and_later_speech_recompute_the_current_deadline() {- let mut session = VoiceSession::default();- session.begin_capture().unwrap();- session.observe_audio(0, 16000, Some(16000)).unwrap();- session.observe_audio(16000, 13 * 16000, None).unwrap();- assert!(!session.silence_elapsed());- session.observe_recognition(true).unwrap();- assert!(session.silence_elapsed());- session- .observe_audio(13 * 16000, 14 * 16000, Some(14 * 16000))- .unwrap();- assert!(!session.silence_elapsed());- assert_eq!(session.phase(), VoicePhase::Listening);- session.observe_audio(14 * 16000, 16 * 16000, None).unwrap();- assert!(!session.silence_elapsed());- session.observe_audio(16 * 16000, 17 * 16000, None).unwrap();- assert!(session.silence_elapsed());+ fn live_grace_ends_with_text_or_timeout_and_later_speech_resets_silence() {+ for has_text in [false, true] {+ let mut session = VoiceSession::default();+ session.begin_capture().unwrap();+ session.observe_audio(0, 32000, Some(24000)).unwrap();+ session.observe_recognition(false).unwrap();+ session.observe_audio(32000, 80000, None).unwrap();+ assert_eq!(session.phase(), VoicePhase::Listening);+ session.observe_audio(80000, 13 * 16000, None).unwrap();+ assert!(!session.silence_elapsed());+ assert_eq!(session.silence_deadline(), Some(15 * 16000));+ let resumed_at = if has_text {+ session.observe_recognition(true).unwrap();+ 13 * 16000+ } else {+ session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();+ 15 * 16000+ };+ assert!(session.silence_elapsed());+ let voice_end = resumed_at + 16000;+ session+ .observe_audio(resumed_at, voice_end, Some(voice_end))+ .unwrap();+ assert!(!session.silence_elapsed());+ assert_eq!(session.phase(), VoicePhase::Listening);+ session+ .observe_audio(voice_end, voice_end + 32000, None)+ .unwrap();+ assert!(!session.silence_elapsed());+ session+ .observe_audio(voice_end + 32000, voice_end + 48000, None)+ .unwrap();+ assert!(session.silence_elapsed());+ }}#[test]