Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Harden mobile dictation lifecycle and bounded transcription

setalpc73247724d25
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index f5822f52d1..537b82b3f6 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -10,6 +10,7 @@ import io.ente.ensu.chat.AttachmentStoreActions
import io.ente.ensu.chat.ChatMessage
import io.ente.ensu.chat.ChatRepository
import io.ente.ensu.chat.ChatStoreActions
+import io.ente.ensu.chat.VoiceInputFailure
import io.ente.ensu.chat.VoiceTranscriptionController
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
import io.ente.ensu.device.ChatDeviceCapability
@@ -91,6 +92,12 @@ class AppStore(
private var composerRevision = 0uL
private var emptyComposerId = UUID.randomUUID().toString()
+
+ private fun invalidateComposer() {
+ composerRevision++
+ voiceController.composerCaret = null
+ }
+
internal val voiceController: VoiceTranscriptionController =
VoiceTranscriptionController(
context.applicationContext,
@@ -167,7 +174,7 @@ class AppStore(
fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)
fun handleMemoryPressure() {
- voiceController.interrupt("Voice input paused because memory is low.")
+ voiceController.interrupt(VoiceInputFailure.MemoryUnavailable)
chatActions.cancelSessionSummary()
modelSettingsActions.handleMemoryPressure()
}
@@ -180,6 +187,10 @@ class AppStore(
fun refreshDeviceCapability(scope: CoroutineScope? = null) {
val capability = deviceCapabilityProvider.chatCapability()
val unsupported = capability is ChatDeviceCapability.UnsupportedLowMemory
+ if (unsupported) {
+ voiceController.discard()
+ invalidateComposer()
+ }
_state.value =
_state.value.copy(
chat =
@@ -234,23 +245,30 @@ class AppStore(
)
}
- fun createNewSession(): String = chatActions.createNewSession()
+ fun createNewSession(): String {
+ voiceController.discard()
+ invalidateComposer()
+ return chatActions.createNewSession()
+ }
fun startNewSessionDraft() {
voiceController.discard()
emptyComposerId = UUID.randomUUID().toString()
- composerRevision++
+ invalidateComposer()
chatActions.startNewSessionDraft()
}
fun selectSession(sessionId: String) {
voiceController.discard()
- composerRevision++
+ invalidateComposer()
chatActions.selectSession(sessionId)
}
fun deleteSession(sessionId: String) {
- if (sessionId == _state.value.chat.currentSessionId) voiceController.discard()
+ if (sessionId == _state.value.chat.currentSessionId) {
+ voiceController.discard()
+ invalidateComposer()
+ }
chatActions.deleteSession(sessionId)
}
@@ -259,8 +277,7 @@ class AppStore(
fun updateMessageText(value: String) {
if (value != _state.value.chat.messageText) {
- composerRevision++
- voiceController.composerCaret = null
+ invalidateComposer()
}
chatActions.updateMessageText(value)
}
@@ -270,11 +287,14 @@ class AppStore(
fun beginEditing(messageId: String) {
voiceController.discard()
- composerRevision++
+ invalidateComposer()
chatActions.beginEditing(messageId)
}
- fun cancelEditing() = chatActions.cancelEditing()
+ fun cancelEditing() {
+ invalidateComposer()
+ chatActions.cancelEditing()
+ }
fun setAttachmentProcessing(isProcessing: Boolean) =
attachmentActions.setAttachmentProcessing(isProcessing)
@@ -319,8 +339,7 @@ class AppStore(
fun sendMessage() {
if (voiceController.active) return
- voiceController.composerCaret = null
- composerRevision++
+ invalidateComposer()
chatActions.sendMessage()
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
index 9f2719038f..e8ea62c6ae 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
@@ -16,7 +16,10 @@ import io.ente.ensu.bindings.needsAssetMigration
import io.ente.ensu.bindings.reconcileKnowledgePack
import io.ente.ensu.bindings.uniffiEnsureInitialized
import java.io.File
+import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
class AssetStore(context: Context) {
@@ -90,6 +93,14 @@ class AssetStore(context: Context) {
val token = CancellationToken()
val lease = AssetDownloadJobService.begin { token.cancel() }
+ val cancellation =
+ launch(Dispatchers.Default, start = CoroutineStart.UNDISPATCHED) {
+ try {
+ awaitCancellation()
+ } finally {
+ token.cancel()
+ }
+ }
try {
core.download(
assets,
@@ -107,7 +118,7 @@ class AssetStore(context: Context) {
token,
)
} finally {
- token.cancel()
+ cancellation.cancel()
AssetDownloadJobService.end(lease)
}
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt
index 19b5e4ebce..5d779963c1 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt
@@ -14,7 +14,6 @@ internal fun captureMicrophoneAudio(
if (isStopped()) return
throw IOException("Microphone read failed")
}
- // Stop changes the route too. Preserve a successful final read while draining.
if (isInterrupted() && !isStopped()) throw IOException("Microphone interrupted")
if (!deliver(count)) return
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index a0fcd2ee21..55a0c40e4e 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -2,7 +2,6 @@ package io.ente.ensu.chat
import android.annotation.SuppressLint
import android.content.Context
-import android.content.pm.ApplicationInfo
import android.media.AudioFormat
import android.media.AudioManager
import android.media.AudioRecord
@@ -11,6 +10,7 @@ import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import io.ente.ensu.assets.AssetStore
+import io.ente.ensu.bindings.AssetDownloadException
import io.ente.ensu.bindings.DictationException
import io.ente.ensu.bindings.DictationFeedResult
import io.ente.ensu.bindings.DictationInsertion
@@ -20,11 +20,12 @@ import io.ente.ensu.bindings.DictationSnapshot
import io.ente.ensu.bindings.DictationStopReason
import io.ente.ensu.bindings.DictationStream
import io.ente.ensu.bindings.DictationStreamCallback
+import io.ente.ensu.bindings.supportsLiveVoice
import io.ente.ensu.bindings.transcriptionModelAsset
import io.ente.ensu.bindings.voiceActivityModelAsset
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
-import io.ente.ensu.device.supportsLiveVoice
import io.ente.ensu.llm.LlmProvider
+import io.ente.ensu.llm.ModelMemoryUnavailable
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.logging.LogLevel
import java.io.IOException
@@ -46,6 +47,36 @@ import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
+internal enum class VoiceInputFailure(val message: String) {
+ Permission("Microphone permission is required. Enable it in Settings."),
+ NoSpeech("No speech detected. Try again."),
+ Download("Voice model download failed. Check your connection and try again."),
+ ModelUnavailable("Voice model could not be loaded. Try again."),
+ StorageFull("Not enough storage space. Free up space and try again."),
+ FileAccess("Could not read or save voice data. Check storage and try again."),
+ MemoryUnavailable("Not enough free memory. Close other apps and try again."),
+ MicrophoneUnavailable("Could not start the microphone. Tap the microphone to try again."),
+ Interrupted("Voice input was interrupted. Review the available text and try again."),
+ ProcessingFailed("Could not transcribe voice input. Review the available text and try again."),
+}
+
+internal fun voiceInputFailure(error: Exception): VoiceInputFailure =
+ when (error) {
+ is ModelMemoryUnavailable -> VoiceInputFailure.MemoryUnavailable
+ is AssetDownloadException.StorageFull,
+ is DictationException.StorageFull -> VoiceInputFailure.StorageFull
+ is AssetDownloadException.Cancelled,
+ is DictationException.Cancelled,
+ is DictationException.InvalidAudio -> VoiceInputFailure.Interrupted
+ is AssetDownloadException.Io,
+ is DictationException.Io,
+ is IOException -> VoiceInputFailure.FileAccess
+ is AssetDownloadException -> VoiceInputFailure.Download
+ is DictationException.NotDownloaded,
+ is DictationException.VadNotDownloaded -> VoiceInputFailure.ModelUnavailable
+ else -> VoiceInputFailure.ProcessingFailed
+ }
+
internal sealed interface VoiceInputState {
data object Idle : VoiceInputState
@@ -57,7 +88,7 @@ internal sealed interface VoiceInputState {
data object Transcribing : VoiceInputState
- data class Error(val message: String) : VoiceInputState
+ data class Error(val reason: VoiceInputFailure) : VoiceInputState
}
internal val VoiceInputState.isRecording: Boolean
@@ -71,20 +102,23 @@ internal class VoiceTranscriptionController(
private val onVoiceInputJob: (Job) -> Unit,
private val onInsert: (DictationInsertion, String) -> Boolean,
) {
- private val diagnosticsEnabled =
- context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0
-
- private fun diagnostic(event: String) {
- if (diagnosticsEnabled) {
- logRepository.log(LogLevel.Info, event, tag = "EnsuVoice")
- }
- }
+ private fun logFailure(error: Exception) =
+ logRepository.log(
+ LogLevel.Error,
+ "Voice input failed",
+ tag = "EnsuVoice",
+ throwable = error,
+ )
private val audioManager = context.getSystemService(AudioManager::class.java)
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
private val usesLiveDictation =
supportsLiveVoice(
- AndroidDeviceCapabilityProvider(context).chatCapability().totalMemoryBytes
+ AndroidDeviceCapabilityProvider(context)
+ .chatCapability()
+ .totalMemoryBytes
+ ?.takeIf { it > 0 }
+ ?.toULong()
)
private val modelAssets =
listOf(
@@ -119,7 +153,7 @@ internal class VoiceTranscriptionController(
var snapshot: DictationSnapshot? by mutableStateOf(null)
private set
- private var notice: String? = null
+ private var failure: VoiceInputFailure? = null
var audioLevels: List<Float> by mutableStateOf(List(24) { 0f })
private set
@@ -141,7 +175,7 @@ internal class VoiceTranscriptionController(
this.insertion = insertion
session = DictationSession(live = usesLiveDictation)
snapshot = session?.snapshot()
- notice = null
+ failure = null
audioLevels = List(24) { 0f }
active = true
state = VoiceInputState.Idle
@@ -166,8 +200,7 @@ internal class VoiceTranscriptionController(
if (foreground) record() else startOnForeground = true
} else {
discard()
- state =
- VoiceInputState.Error("Microphone permission is required. Enable it in Settings.")
+ state = VoiceInputState.Error(VoiceInputFailure.Permission)
}
}
@@ -184,7 +217,7 @@ internal class VoiceTranscriptionController(
fun record() {
val owner = session ?: return
if (!active || !foreground || hasActiveWork || attemptFinished) return
- notice = null
+ failure = null
state = VoiceInputState.Preparing
val token = generation
hasActiveWork = true
@@ -223,18 +256,29 @@ internal class VoiceTranscriptionController(
audioLevels = audioLevels.drop(1) + level
}
},
- ) {
+ ) { error ->
scope.launch {
- if (token == generation)
- interrupt(
- "Microphone interrupted. Review the text before continuing."
- )
+ if (token == generation) {
+ logFailure(error)
+ interrupt()
+ }
}
}
capture = microphone
ownedCapture = microphone
coroutineScope {
- val hardwareReady = async(Dispatchers.IO) { microphone.prepare() }
+ val hardwareReady =
+ async(Dispatchers.IO) {
+ try {
+ microphone.prepare()
+ } catch (error: Exception) {
+ withContext(Dispatchers.Main.immediate) {
+ if (token == generation && failure == null)
+ failure = VoiceInputFailure.MicrophoneUnavailable
+ }
+ throw error
+ }
+ }
val callback =
object : DictationStreamCallback {
override fun onReady() {
@@ -254,13 +298,17 @@ internal class VoiceTranscriptionController(
microphone.start(scope)
state = VoiceInputState.Recording
audioLevels = List(24) { 0f }
- notice = null
+ failure = null
refresh(owner, token)
- } catch (_: Exception) {
- if (token == generation)
+ } catch (error: CancellationException) {
+ throw error
+ } catch (error: Exception) {
+ if (token == generation) {
+ logFailure(error)
interrupt(
- "Could not start the microphone. Tap the microphone to start again."
+ VoiceInputFailure.MicrophoneUnavailable
)
+ }
}
}
}
@@ -279,16 +327,18 @@ internal class VoiceTranscriptionController(
llmProvider.runLiveVoice(activeStream, callback)
}
} catch (error: CancellationException) {
+ if (token == generation && failure == null)
+ failure = VoiceInputFailure.Interrupted
throw error
} catch (error: Exception) {
ownedStream?.let { runCatching { it.abandon(DictationStopReason.FAILURE) } }
- diagnostic(
- if (error is DictationException.InvalidOperation)
- "failure ${error.detail}"
- else "failure ${error.javaClass.simpleName}"
+ if (
+ error !is AssetDownloadException.Cancelled &&
+ error !is DictationException.Cancelled
)
- if (token == generation && notice == null) {
- notice = "Voice input stopped. Review the available text or try again."
+ logFailure(error)
+ if (token == generation && failure == null) {
+ failure = voiceInputFailure(error)
}
} finally {
withContext(NonCancellable) { ownedCapture?.stopAndDrain() }
@@ -320,8 +370,8 @@ internal class VoiceTranscriptionController(
val completedState =
when {
added && !value.incomplete -> VoiceInputState.Idle
- text.isBlank() -> VoiceInputState.Error(notice ?: "No speech detected.")
- else -> VoiceInputState.Error("Voice input stopped.")
+ text.isBlank() -> VoiceInputState.Error(failure ?: VoiceInputFailure.NoSpeech)
+ else -> VoiceInputState.Error(failure ?: VoiceInputFailure.Interrupted)
}
active = false
isFinishingSlow = false
@@ -374,19 +424,19 @@ internal class VoiceTranscriptionController(
}
}
- fun interrupt(message: String = "Voice input stopped.") {
+ fun interrupt(reason: VoiceInputFailure = VoiceInputFailure.Interrupted) {
if (!active || attemptFinished) return
if (!hasActiveWork && stream == null) {
discard()
return
}
+ failure = reason
+ state = VoiceInputState.Transcribing
capture?.stop()
runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }
work?.cancel()
finishingNoticeJob?.cancel()
isFinishingSlow = false
- notice = message
- state = VoiceInputState.Transcribing
}
fun discard() {
@@ -418,7 +468,7 @@ private class LiveMicrophone(
private val stream: DictationStream,
private val audioManager: AudioManager,
private val onLevel: (Float) -> Unit,
- private val onInterrupted: () -> Unit,
+ private val onInterrupted: (Exception) -> Unit,
) {
private val stopped = AtomicBoolean(false)
@Volatile private var recorder: AudioRecord? = null
@@ -501,10 +551,10 @@ private class LiveMicrophone(
)
stream.feed(buffer) == DictationFeedResult.ACCEPTED
}
- } catch (_: Exception) {
+ } catch (error: Exception) {
if (!stopped.get()) {
runCatching { stream.abandon(DictationStopReason.FAILURE) }
- onInterrupted()
+ onInterrupted(error)
}
} finally {
stop()
@@ -517,7 +567,6 @@ private class LiveMicrophone(
suspend fun stopAndDrain() {
stop()
readJob?.join()
- // A reader canceled before dispatch never enters its cleanup block.
recorder?.release()
recorder = null
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
index 4974e98aa2..dd79663204 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
@@ -56,8 +56,7 @@ import io.ente.ensu.designsystem.HugeIcons
internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
val state = controller.state
if (!controller.active && state !is VoiceInputState.Error) return
- val permissionNeeded =
- (state as? VoiceInputState.Error)?.message?.contains("permission") == true
+ val permissionNeeded = (state as? VoiceInputState.Error)?.reason == VoiceInputFailure.Permission
val context = LocalContext.current
val label =
when (state) {
@@ -69,9 +68,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
VoiceInputState.Recording ->
if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"
else "Listening"
- is VoiceInputState.Error ->
- if (permissionNeeded) "Microphone access"
- else if (state.message == "No speech detected.") "No speech" else "Stopped"
+ is VoiceInputState.Error -> state.reason.message
}
Row(
Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
index 832c392269..6bbb678100 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
@@ -2,9 +2,6 @@ package io.ente.ensu.device
const val CHAT_MIN_RAM_BYTES: Long = 1_600_000_000L
-internal fun supportsLiveVoice(totalMemoryBytes: Long?): Boolean =
- totalMemoryBytes != null && totalMemoryBytes > 4L * 1024 * 1024 * 1024
-
internal fun chatDeviceCapability(totalMemoryBytes: Long): ChatDeviceCapability =
when {
totalMemoryBytes <= 0 -> ChatDeviceCapability.Unknown
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt
index aeebb06bd9..82fbceeeea 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt
@@ -12,7 +12,6 @@ class MicrophoneCaptureTest {
isStopped = { stopped },
read = { 3200 },
isInterrupted = {
- // AudioRecord.stop() clears the route after a successful read.
stopped = true
true
},
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
index 5ba9a19595..d1480cefd7 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
@@ -7,15 +7,6 @@ import org.junit.Assert.assertTrue
import org.junit.Test
class ChatDeviceCapabilityTest {
- @Test
- fun liveVoiceRequiresMoreThanFourGiB() {
- val limit = 4L * 1024 * 1024 * 1024
- for (bytes in listOf(null, -1L, 0L, 4_000_000_000L, limit)) {
- assertFalse(supportsLiveVoice(bytes))
- }
- for (bytes in listOf(limit + 1, 8_000_000_000L)) assertTrue(supportsLiveVoice(bytes))
- }
-
@Test
fun minimumRamBlocksOnlyKnownReadingsBelowTheFloor() {
val belowMinimum = chatDeviceCapability(1_599_999_999L)
diff --git a/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj b/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj
index 85e1b32b02..a18b68369f 100644
--- a/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj
+++ b/apple/apps/ensu/Ensu.xcodeproj/project.pbxproj
@@ -7,6 +7,7 @@
objects = {
/* Begin PBXBuildFile section */
+ A70000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A70000022F11111100AA0001; };
A50000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A50000022F11111100AA0001; };
A40000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A40000022F11111100AA0001; };
A30000012F11111100AA0001 = {isa = PBXBuildFile; fileRef = A30000022F11111100AA0001; };
@@ -29,6 +30,7 @@
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
+ A70000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceCaptureDeliveryTests.swift; sourceTree = "<group>"; };
A50000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KnowledgePreferencesTests.swift; sourceTree = "<group>"; };
A40000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelMemoryCoordinationTests.swift; sourceTree = "<group>"; };
A30000022F11111100AA0001 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageMarkdownTests.swift; sourceTree = "<group>"; };
@@ -85,6 +87,7 @@
isa = PBXGroup;
children = (
A10000022F11111100AA0001 /* LlmIntegrationTests.swift */,
+ A70000022F11111100AA0001,
A40000022F11111100AA0001,
A50000022F11111100AA0001,
A20000022F11111100AA0001,
@@ -256,6 +259,7 @@
buildActionMask = 2147483647;
files = (
A10000012F11111100AA0001 /* LlmIntegrationTests.swift in Sources */,
+ A70000012F11111100AA0001,
A40000012F11111100AA0001,
A50000012F11111100AA0001,
A20000012F11111100AA0001,
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index d7711f92f7..fe2100e4fd 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -272,7 +272,7 @@ final class ChatViewModel: ObservableObject {
}
func handleMemoryPressure() {
- voiceTranscriber.interrupt(message: "Voice input paused because memory is low.")
+ voiceTranscriber.interrupt(reason: .memoryUnavailable)
sessionSummaryTask?.cancel()
warmupSuppressed = true
cancelChatWarmup()
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
index 260fb7715c..021cfc7f05 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
@@ -11,35 +11,48 @@ struct VoiceInputControls: View {
case .preparing: return "Preparing"
case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"
case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"
- case .error(let message):
- if message.contains("permission") { return "Microphone access" }
- return message == "No speech detected." ? "No speech" : "Stopped"
+ case .error(.permission): return "Microphone access"
+ case .error(.noSpeech): return "No speech"
+ case .error: return "Stopped"
case .idle: return "Preparing"
}
}
+ private var failure: VoiceInputFailure? {
+ if case .error(let failure) = service.state { return failure }
+ return service.notice
+ }
+
var body: some View {
if service.active || service.state.isError {
- HStack(spacing: 12) {
- Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)
- .accessibilityHint(service.notice ?? "")
- .onTapGesture {
- if case .error(let message) = service.state,
- message.contains("permission"),
- let url = URL(string: UIApplication.openSettingsURLString)
- {
- openURL(url)
- }
+ VStack(alignment: .leading, spacing: EnsuSpacing.xs) {
+ HStack(spacing: 12) {
+ Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)
+ if service.state.isRecording {
+ waveform.frame(height: 24)
+ } else {
+ Spacer(minLength: 0)
+ }
+ if service.active, !service.state.isRecording {
+ Button("Cancel", action: service.cancel)
+ .font(EnsuTypography.small)
+ .foregroundStyle(EnsuColor.textPrimary)
}
- if service.state.isRecording {
- waveform.frame(height: 24)
- } else {
- Spacer(minLength: 0)
}
- if service.active, !service.state.isRecording {
- Button("Cancel", action: service.cancel)
+ if let failure {
+ Text(failure.message)
+ .font(EnsuTypography.small)
+ .foregroundStyle(EnsuColor.textMuted)
+ .fixedSize(horizontal: false, vertical: true)
+ if failure == .permission {
+ Button("Open Settings") {
+ if let url = URL(string: UIApplication.openSettingsURLString) {
+ openURL(url)
+ }
+ }
.font(EnsuTypography.small)
.foregroundStyle(EnsuColor.textPrimary)
+ }
}
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 897e4b98e3..961975aa3d 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -2,13 +2,69 @@ import Foundation
import AVFoundation
import Combine
+enum VoiceInputFailure: Equatable {
+ case permission, microphone, noSpeech, interrupted, captureInterrupted
+ case modelUnavailable, memoryUnavailable, storageFull, download, fileAccess, transcription
+ case insertion
+
+ var message: String {
+ switch self {
+ case .permission:
+ return "Microphone permission is required. Enable it in Settings."
+ case .microphone:
+ return "Could not start the microphone. Tap the microphone to try again."
+ case .noSpeech:
+ return "No speech detected. Tap the microphone to try again."
+ case .interrupted:
+ return "Voice input stopped. Review the available text or try again."
+ case .captureInterrupted:
+ return "Audio capture interrupted. Some words may be missing; review the text."
+ case .modelUnavailable:
+ return "The voice model is unavailable. Tap the microphone to download it again."
+ case .memoryUnavailable:
+ return "Not enough free memory for voice input. Close other apps and try again."
+ case .storageFull:
+ return "Not enough storage for voice input. Free up space and try again."
+ case .download:
+ return "Could not download the voice model. Check your connection and try again."
+ case .fileAccess:
+ return "Could not access voice model files. Try again."
+ case .transcription:
+ return "Could not transcribe voice input. Review the available text or try again."
+ case .insertion:
+ return "The draft changed before voice input finished. Tap the microphone to try again."
+ }
+ }
+
+ static func classify(_ error: Error) -> Self? {
+ switch error {
+ case is CancellationError, AssetDownloadError.Cancelled, DictationError.Cancelled:
+ return nil
+ case is ModelMemoryUnavailable:
+ return .memoryUnavailable
+ case DictationError.NotDownloaded, DictationError.VadNotDownloaded:
+ return .modelUnavailable
+ case AssetDownloadError.StorageFull, DictationError.StorageFull:
+ return .storageFull
+ case AssetDownloadError.Io, DictationError.Io:
+ return .fileAccess
+ case is AssetDownloadError:
+ return .download
+ case DictationError.InvalidAudio:
+ return .captureInterrupted
+ default:
+ return .transcription
+ }
+ }
+}
+
enum VoiceInputState: Equatable {
case idle
case preparing
case recording
case downloading(percent: Int?)
case transcribing
- case error(String)
+ case error(VoiceInputFailure)
var isRecording: Bool {
self == .recording
@@ -30,7 +86,7 @@ final class VoiceTranscriptionService: ObservableObject {
}
@Published private(set) var snapshot: DictationSnapshot?
@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)
- @Published private(set) var notice: String?
+ @Published private(set) var notice: VoiceInputFailure?
@Published private(set) var state: VoiceInputState = .idle
@Published private(set) var isFinishingSlow = false
var onActivityChanged: (@MainActor () -> Void)?
@@ -39,6 +95,7 @@ final class VoiceTranscriptionService: ObservableObject {
private let assetStore: AssetStore
private let modelAssets: [Asset]
private let usesLiveDictation: Bool
+ private let logger = EnsuLogging.shared.logger("EnsuVoice")
private var session: DictationSession?
private var stream: DictationStream?
private var recorder: LivePcmRecorder?
@@ -56,14 +113,15 @@ final class VoiceTranscriptionService: ObservableObject {
}
private func diagnostic(_ event: String) {
#if DEBUG
- EnsuLogging.shared.logger("EnsuVoice").info(event)
+ logger.info(event)
#endif
}
init(provider: LlmProvider, assetStore: AssetStore) {
self.provider = provider
self.assetStore = assetStore
- usesLiveDictation = supportsLiveVoice()
+ usesLiveDictation = supportsLiveVoice(
+ totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)
modelAssets = [
transcriptionModelAsset(),
voiceActivityModelAsset(),
@@ -149,15 +207,14 @@ final class VoiceTranscriptionService: ObservableObject {
if self.foreground { self.record() } else { self.startOnForeground = true }
} else {
self.cancel()
- self.state = .error(
- "Microphone permission is required. Enable it in Settings.")
+ self.state = .error(.permission)
}
}
}
return
case .denied:
cancel()
- state = .error("Microphone permission is required. Enable it in Settings.")
+ state = .error(.permission)
return
case .granted: break
@unknown default: return
@@ -195,7 +252,14 @@ final class VoiceTranscriptionService: ObservableObject {
try Task.checkCancellation()
guard self.generation == token, self.foreground else { return }
self.state = .preparing
- let recorder = try LivePcmRecorder()
+ let recorder: LivePcmRecorder
+ do {
+ recorder = try LivePcmRecorder()
+ } catch {
+ self.logger.error("Could not prepare the microphone", error)
+ self.notice = .microphone
+ return
+ }
self.recorder = recorder
let sampleRate = recorder.sampleRate
let stream = try DictationStream(
@@ -212,13 +276,15 @@ final class VoiceTranscriptionService: ObservableObject {
}
}
) {
- [weak self] in
+ [weak self] error in
Task { @MainActor [weak self] in
guard let self, self.generation == token else { return }
- self.interrupt(
- message:
- "Audio capture interrupted. Some words may be missing; review the text."
- )
+ if let error {
+ self.logger.error("Audio capture failed", error)
+ } else {
+ self.logger.warning("Audio capture exceeded its buffer limit")
+ }
+ self.interrupt(reason: .captureInterrupted)
}
}
let callback = LiveVoiceCallback(
@@ -236,10 +302,8 @@ final class VoiceTranscriptionService: ObservableObject {
self.notice = nil
self.refresh(owner, token: token)
} catch {
- self.interrupt(
- message:
- "Could not start the microphone. Tap the microphone to start again."
- )
+ self.logger.error("Could not start the microphone", error)
+ self.interrupt(reason: .microphone)
}
}
},
@@ -257,16 +321,11 @@ final class VoiceTranscriptionService: ObservableObject {
}
)
try await self.provider.runLiveVoice(stream: stream, callback: callback)
- } catch is CancellationError {
} catch {
- if let error = error as? DictationError, case .InvalidOperation(let detail) = error
- {
- self.diagnostic("failure \(detail)")
- } else {
- self.diagnostic("failure \(type(of: error))")
- }
+ let failure = VoiceInputFailure.classify(error)
+ if failure != nil { self.logger.error("Voice input failed", error) }
if self.generation == token, self.notice == nil {
- self.notice = "Voice input stopped. Review the available text or try again."
+ self.notice = failure ?? .interrupted
}
}
}
@@ -286,10 +345,9 @@ final class VoiceTranscriptionService: ObservableObject {
let added = hasText && insertion.map { onInsert?($0, text) == true } == true
let completedState: VoiceInputState
if added {
- completedState = value.incomplete ? .error("Voice input stopped.") : .idle
+ completedState = value.incomplete ? .error(notice ?? .interrupted) : .idle
} else {
- completedState = .error(
- hasText ? "Voice input stopped." : notice ?? "No speech detected.")
+ completedState = .error(hasText ? .insertion : notice ?? .noSpeech)
}
active = false
isFinishingSlow = false
@@ -327,7 +385,7 @@ final class VoiceTranscriptionService: ObservableObject {
if recorder == nil { try? stream?.stop(reason: reason) }
}
- func interrupt(message: String = "Voice input stopped.") {
+ func interrupt(reason: VoiceInputFailure = .interrupted) {
guard active, !attemptFinished else { return }
if !hasActiveTasks, stream == nil {
cancel()
@@ -338,7 +396,7 @@ final class VoiceTranscriptionService: ObservableObject {
task?.cancel()
finishingNoticeTask?.cancel()
isFinishingSlow = false
- notice = message
+ notice = reason
state = .transcribing
}
@@ -372,7 +430,9 @@ private final class LiveVoiceCallback: DictationStreamCallback, Sendable {
change: @escaping @Sendable () -> Void,
closed: @escaping @Sendable () -> Void
) {
- self.ready = ready; self.change = change; self.closed = closed
+ self.ready = ready
+ self.change = change
+ self.closed = closed
}
func onReady() { ready() }
func onChange() { change() }
@@ -416,7 +476,10 @@ private final class LivePcmRecorder {
}
func stop(afterDelivery: (@Sendable () -> Void)? = nil) {
- if installed { engine.inputNode.removeTap(onBus: 0); installed = false }
+ if installed {
+ engine.inputNode.removeTap(onBus: 0)
+ installed = false
+ }
delivery?.close()
engine.stop()
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
@@ -431,7 +494,7 @@ private final class VoiceNotificationTokens: @unchecked Sendable {
deinit { for value in values { NotificationCenter.default.removeObserver(value) } }
}
-private final class LiveCaptureDelivery: @unchecked Sendable {
+final class LiveCaptureDelivery: @unchecked Sendable {
private let queue = DispatchQueue(label: "io.ente.ensu.voice.capture", qos: .userInitiated)
private let lock = NSLock()
private var accepting = true
@@ -439,49 +502,69 @@ private final class LiveCaptureDelivery: @unchecked Sendable {
private var chunksPending = 0
private let stream: DictationStream
private let rate: Int
- private let interrupted: @Sendable () -> Void
+ private let interrupted: @Sendable (Error?) -> Void
private let level: @Sendable (Float) -> Void
private var meterFrames = 0
private var meterEnergy = 0.0
init(
stream: DictationStream, sampleRate: UInt32,
- level: @escaping @Sendable (Float) -> Void, interrupted: @escaping @Sendable () -> Void
+ level: @escaping @Sendable (Float) -> Void,
+ interrupted: @escaping @Sendable (Error?) -> Void
) {
- self.stream = stream; rate = Int(sampleRate); self.interrupted = interrupted
+ self.stream = stream
+ rate = Int(sampleRate)
+ self.interrupted = interrupted
self.level = level
}
- func close() { lock.lock(); accepting = false; lock.unlock() }
+
+ func close() {
+ lock.lock()
+ accepting = false
+ lock.unlock()
+ }
+
func drain(_ completion: @escaping @Sendable () -> Void) { queue.async(execute: completion) }
func accept(_ buffer: AVAudioPCMBuffer) {
let frames = Int(buffer.frameLength)
+ accept(frames: frames) {
+ guard let channels = buffer.floatChannelData, buffer.format.channelCount > 0 else {
+ return nil
+ }
+ let channelCount = Int(buffer.format.channelCount)
+ var mono = [Float](repeating: 0, count: frames)
+ for frame in 0..<frames {
+ for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }
+ mono[frame] /= Float(channelCount)
+ }
+ return mono
+ }
+ }
+
+ func accept(frames: Int, copySamples: () -> [Float]?) {
guard frames > 0 else { return }
lock.lock()
- guard accepting else { lock.unlock(); return }
+ defer { lock.unlock() }
+ guard accepting else { return }
guard frames <= rate, framesPending + frames <= rate * 2, chunksPending < 128,
- let channels = buffer.floatChannelData, buffer.format.channelCount > 0
+ let samples = copySamples()
else {
accepting = false
- lock.unlock()
queue.async { [self] in
- try? stream.abandon(reason: .failure); interrupted()
+ try? stream.abandon(reason: .failure)
+ interrupted(nil)
}
return
}
framesPending += frames
chunksPending += 1
- lock.unlock()
- let channelCount = Int(buffer.format.channelCount)
- var mono = [Float](repeating: 0, count: frames)
- for frame in 0..<frames {
- for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }
- mono[frame] /= Float(channelCount)
- }
- let samples = mono
queue.async { [self] in
defer {
- lock.lock(); framesPending -= frames; chunksPending -= 1; lock.unlock()
+ lock.lock()
+ framesPending -= frames
+ chunksPending -= 1
+ lock.unlock()
}
do {
meterFrames += frames
@@ -489,7 +572,8 @@ private final class LiveCaptureDelivery: @unchecked Sendable {
if meterFrames >= rate / 10 {
let rms = sqrt(meterEnergy / Double(meterFrames))
level(Float(min(1, max(0, (20 * log10(max(rms, 0.00001)) + 55) / 45))))
- meterFrames = 0; meterEnergy = 0
+ meterFrames = 0
+ meterEnergy = 0
}
for start in stride(from: 0, to: frames, by: rate / 10) {
let end = min(frames, start + rate / 10)
@@ -509,8 +593,14 @@ private final class LiveCaptureDelivery: @unchecked Sendable {
}
}
} catch {
- lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()
- if wasAccepting { try? stream.abandon(reason: .failure); interrupted() }
+ lock.lock()
+ let wasAccepting = accepting
+ accepting = false
+ lock.unlock()
+ if wasAccepting {
+ try? stream.abandon(reason: .failure)
+ interrupted(error)
+ }
}
}
}
diff --git a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
index 44292bdc0d..23b45c5b68 100644
--- a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
+++ b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
@@ -26,10 +26,6 @@ enum ChatDeviceCapability: Equatable {
}
}
-func supportsLiveVoice(totalMemoryBytes: UInt64 = ProcessInfo.processInfo.physicalMemory) -> Bool {
- totalMemoryBytes > 4 * 1024 * 1024 * 1024
-}
-
func currentChatDeviceCapability() -> ChatDeviceCapability {
chatDeviceCapability(totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)
}
diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
index 2765c1b472..cc3ad618bd 100644
--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
@@ -20,16 +20,6 @@ private actor ModelOperationBarrier {
@MainActor
final class ModelMemoryCoordinationTests: XCTestCase {
- func testLiveVoiceRequiresMoreThanFourGiB() {
- let limit: UInt64 = 4 * 1024 * 1024 * 1024
- for bytes: UInt64 in [0, 4_000_000_000, limit] {
- XCTAssertFalse(supportsLiveVoice(totalMemoryBytes: bytes))
- }
- for bytes: UInt64 in [limit + 1, 8_000_000_000] {
- XCTAssertTrue(supportsLiveVoice(totalMemoryBytes: bytes))
- }
- }
-
func testMinimumRamBlocksOnlyKnownReadingsBelowTheFloor() {
XCTAssertFalse(chatDeviceCapability(totalMemoryBytes: 1_599_999_999).isChatSupported)
XCTAssertTrue(chatDeviceCapability(totalMemoryBytes: 1_600_000_000).isChatSupported)
diff --git a/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swift b/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swift
new file mode 100644
index 0000000000..3588c7d767
--- /dev/null
+++ b/apple/apps/ensu/EnsuTests/VoiceCaptureDeliveryTests.swift
@@ -0,0 +1,56 @@
+import Foundation
+import XCTest
+@testable import Ensu
+
+@MainActor
+final class VoiceCaptureDeliveryTests: XCTestCase {
+ func testStopDrainsAnAcceptedBufferBeforeClosingTheStream() async throws {
+ let owner = DictationSession(live: false)
+ let stream = try DictationStream(session: owner, sampleRate: 16000, channels: 1)
+ defer { try? stream.abandon(reason: .discard) }
+ let copying = expectation(description: "audio copy started")
+ let stopping = expectation(description: "stop requested")
+ let delivered = expectation(description: "audio delivered")
+ let drained = expectation(description: "capture drained")
+ let releaseCopy = DispatchSemaphore(value: 0)
+ let drainCompleted = DispatchSemaphore(value: 0)
+ defer { releaseCopy.signal() }
+ let delivery = LiveCaptureDelivery(
+ stream: stream, sampleRate: 16000,
+ level: { _ in delivered.fulfill() },
+ interrupted: { error in
+ XCTFail("Unexpected capture failure: \(String(describing: error))")
+ }
+ )
+
+ DispatchQueue.global().async {
+ delivery.accept(frames: 1600) {
+ copying.fulfill()
+ guard releaseCopy.wait(timeout: .now() + 5) == .success else {
+ XCTFail("Audio copy was not released")
+ return nil
+ }
+ return Array(repeating: 0.25, count: 1600)
+ }
+ }
+ await fulfillment(of: [copying], timeout: 2)
+ DispatchQueue.global().async {
+ stopping.fulfill()
+ delivery.close()
+ delivery.drain {
+ do { try stream.stop(reason: .done) } catch { XCTFail("\(error)") }
+ drained.fulfill()
+ drainCompleted.signal()
+ }
+ }
+ await fulfillment(of: [stopping], timeout: 2)
+ XCTAssertEqual(drainCompleted.wait(timeout: .now() + 0.1), .timedOut)
+ releaseCopy.signal()
+ await fulfillment(of: [delivered, drained], timeout: 2, enforceOrder: true)
+ XCTAssertEqual(try stream.feed(pcmLe: Data([0, 0])), .closed)
+ delivery.accept(frames: 1600) {
+ XCTFail("Closed capture accepted another buffer")
+ return nil
+ }
+ }
+}
diff --git a/rust/bindings/uniffi/ensu/src/config.rs b/rust/bindings/uniffi/ensu/src/config.rs
index 3ca8359a02..0c8f35ef94 100644
--- a/rust/bindings/uniffi/ensu/src/config.rs
+++ b/rust/bindings/uniffi/ensu/src/config.rs
@@ -165,6 +165,11 @@ pub fn config_defaults() -> ConfigDefaults {
config::defaults().into()
}
+#[uniffi::export]
+pub fn supports_live_voice(total_memory_bytes: Option<u64>) -> bool {
+ config::supports_live_voice(total_memory_bytes)
+}
+
#[uniffi::export]
pub fn resolve_model_policy(
surface: ModelRuntimeSurface,
diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rs
index 90683bce04..04e1fb4421 100644
--- a/rust/bindings/uniffi/ensu/src/voice.rs
+++ b/rust/bindings/uniffi/ensu/src/voice.rs
@@ -6,14 +6,37 @@ use std::time::{Duration, Instant};
#[derive(Debug, thiserror::Error, uniffi::Error)]
pub enum DictationError {
+ #[error("Transcription model is not downloaded")]
+ NotDownloaded,
+ #[error("Voice activity model is not downloaded")]
+ VadNotDownloaded,
+ #[error("not enough storage space")]
+ StorageFull,
+ #[error("Voice recording was cancelled")]
+ Cancelled,
+ #[error("{detail}")]
+ InvalidAudio { detail: String },
+ #[error("{detail}")]
+ Io { detail: String },
+ #[error("{detail}")]
+ ProcessingFailed { detail: String },
#[error("{detail}")]
InvalidOperation { detail: String },
}
impl From<voice::VoiceError> for DictationError {
fn from(value: voice::VoiceError) -> Self {
- Self::InvalidOperation {
- detail: value.to_string(),
+ match value {
+ voice::VoiceError::NotDownloaded => Self::NotDownloaded,
+ voice::VoiceError::VadNotDownloaded => Self::VadNotDownloaded,
+ voice::VoiceError::StorageFull => Self::StorageFull,
+ voice::VoiceError::Cancelled => Self::Cancelled,
+ voice::VoiceError::InvalidAudio(detail) => Self::InvalidAudio { detail },
+ voice::VoiceError::Io(detail) => Self::Io { detail },
+ voice::VoiceError::ProcessingFailed(detail) => Self::ProcessingFailed { detail },
+ error => Self::InvalidOperation {
+ detail: error.to_string(),
+ },
}
}
}
@@ -333,48 +356,6 @@ mod tests {
}
}
- #[test]
- fn composer_previews_replace_the_selection_without_consuming_the_transfer() {
- let insertion = DictationInsertion::new("chat".into(), 7, 7, 13);
- let original = " meet Friday\n".to_owned();
- assert_eq!(
- insertion
- .preview(original.clone(), String::new())
- .unwrap()
- .text,
- original
- );
- assert_eq!(
- insertion
- .preview(original.clone(), "Sat".into())
- .unwrap()
- .text,
- " meet Sat\n"
- );
- let preview = insertion
- .preview(original.clone(), "Saturday".into())
- .unwrap();
- assert_eq!(preview.revision, 7);
- let final_text = insertion
- .apply("chat".into(), 7, original, "Saturday".into(), true, false)
- .unwrap();
- assert_eq!(preview.text, final_text.text);
- assert_eq!(preview.caret_utf16, final_text.caret_utf16);
- assert_eq!(final_text.revision, 8);
- assert!(
- insertion
- .apply(
- "chat".into(),
- 8,
- final_text.text,
- "Sunday".into(),
- true,
- false
- )
- .is_err()
- );
- }
-
fn event(session: &DictationSession, event: LiveEvent) {
session.lock().apply(event, || true).unwrap();
}
@@ -414,18 +395,6 @@ mod tests {
assert!(!value.worker_active);
assert!(value.incomplete);
assert!(matches!(value.phase, DictationPhase::Ready));
- let insertion = DictationInsertion::new("chat".into(), 0, 7, 7);
- let draft = insertion
- .apply(
- "chat".into(),
- 0,
- "Notes: ".into(),
- value.text + &value.hypothesis,
- true,
- false,
- )
- .unwrap();
- assert_eq!(draft.text, "Notes: Keep this opening word");
assert!(matches!(
owner
.lock()
@@ -453,14 +422,13 @@ mod tests {
let owner = DictationSession::new(live);
let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
let signals = Arc::new(StreamSignals::default());
- assert!(
- stream
- .run(
- crate::transcription::Transcriber::new("missing".into(), "missing".into()),
- signals.clone()
- )
- .is_err()
- );
+ assert!(matches!(
+ stream.run(
+ crate::transcription::Transcriber::new("missing".into(), "missing".into()),
+ signals.clone()
+ ),
+ Err(DictationError::VadNotDownloaded)
+ ));
assert_eq!(signals.ready.load(Ordering::Relaxed), 0);
assert_eq!(signals.stopped.load(Ordering::Relaxed), 1);
assert!(!owner.snapshot().worker_active);
diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rs
index 197badaa6f..34047a748e 100644
--- a/rust/crates/ensu/src/config.rs
+++ b/rust/crates/ensu/src/config.rs
@@ -206,6 +206,10 @@ pub(crate) fn llm_catalog() -> Vec<ModelPreset> {
]
}
+pub fn supports_live_voice(total_memory_bytes: Option<u64>) -> bool {
+ total_memory_bytes.is_some_and(|bytes| bytes > MOBILE_SMALL_MEMORY_MAX_BYTES)
+}
+
pub fn resolve_model_policy(
surface: ModelRuntimeSurface,
total_memory_bytes: Option<u64>,
@@ -370,6 +374,21 @@ mod tests {
}
}
+ #[test]
+ fn live_voice_requires_more_than_four_gib() {
+ let four_gib = 4 * 1024 * 1024 * 1024;
+ for (memory, expected) in [
+ (None, false),
+ (Some(0), false),
+ (Some(four_gib - 1), false),
+ (Some(four_gib), false),
+ (Some(four_gib + 1), true),
+ (Some(8 * 1024 * 1024 * 1024), true),
+ ] {
+ assert_eq!(supports_live_voice(memory), expected, "RAM: {memory:?}");
+ }
+ }
+
#[test]
fn mobile_ram_ladder_enforces_inclusive_boundaries_and_saved_preferences() {
let gib = 1024 * 1024 * 1024;
diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rs
index 0b3d1b9083..faa79cf67c 100644
--- a/rust/crates/ensu/src/transcription/audio.rs
+++ b/rust/crates/ensu/src/transcription/audio.rs
@@ -17,6 +17,7 @@ pub fn extract_speech_from_pcm16(
vad_model_path: impl AsRef<Path>,
input_sample_rate: u32,
pcm_le: &[u8],
+ is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
if input_sample_rate == 0 {
return Err(TranscriptionError::InvalidAudio(
@@ -36,13 +37,14 @@ pub fn extract_speech_from_pcm16(
.map(|chunk| i16::from_le_bytes(*chunk) as f32 / 32768.0)
.collect::<Vec<_>>();
- extract_speech(vad_model_path, input_sample_rate, &samples)
+ extract_speech(vad_model_path, input_sample_rate, &samples, is_cancelled)
}
fn extract_speech(
vad_model_path: impl AsRef<Path>,
input_sample_rate: u32,
samples: &[f32],
+ is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
if samples.is_empty() {
return Ok(Vec::new());
@@ -60,13 +62,14 @@ fn extract_speech(
VAD_ONSET_FRAMES,
);
- extract_speech_with_vad(input_sample_rate, samples, &mut vad)
+ extract_speech_with_vad(input_sample_rate, samples, &mut vad, is_cancelled)
}
fn extract_speech_with_vad(
input_sample_rate: u32,
samples: &[f32],
vad: &mut dyn Vad,
+ is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
let mut resampler = FrameResampler::new(
input_sample_rate as usize,
@@ -88,8 +91,16 @@ fn extract_speech_with_vad(
Err(error) => vad_error = Some(error),
}
};
- resampler.push(samples, &mut emit);
- resampler.finish(&mut emit);
+ for chunk in samples.chunks(input_sample_rate as usize) {
+ if is_cancelled() {
+ return Ok(Vec::new());
+ }
+ resampler.push(chunk, &mut emit)?;
+ }
+ if is_cancelled() {
+ return Ok(Vec::new());
+ }
+ resampler.finish(&mut emit)?;
if let Some(error) = vad_error {
return Err(error.into());
}
@@ -143,10 +154,10 @@ impl FrameResampler {
})
}
- fn push(&mut self, mut src: &[f32], mut emit: impl FnMut(&[f32])) {
+ fn push(&mut self, mut src: &[f32], mut emit: impl FnMut(&[f32])) -> Result<()> {
if self.resampler.is_none() {
self.emit_frames(src, &mut emit);
- return;
+ return Ok(());
}
while !src.is_empty() {
@@ -156,27 +167,38 @@ impl FrameResampler {
src = &src[take..];
if self.in_buf.len() == self.chunk_in {
- if let Some(resampler) = self.resampler.as_mut()
- && let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)
- && let Ok(out) = resampler.process(&input, None)
- {
+ if let Some(resampler) = self.resampler.as_mut() {
+ let input =
+ InterleavedSlice::new(&self.in_buf, 1, self.chunk_in).map_err(|error| {
+ TranscriptionError::InvalidAudio(format!(
+ "Prepare audio resampler input: {error}"
+ ))
+ })?;
+ let out = resampler.process(&input, None).map_err(|error| {
+ TranscriptionError::Transcribe(format!("Resample recorded audio: {error}"))
+ })?;
self.emit_frames(&out.take_data(), &mut emit);
}
self.in_buf.clear();
}
}
+ Ok(())
}
- fn finish(&mut self, mut emit: impl FnMut(&[f32])) {
+ fn finish(&mut self, mut emit: impl FnMut(&[f32])) -> Result<()> {
if let Some(resampler) = self.resampler.as_mut()
&& !self.in_buf.is_empty()
{
self.in_buf.resize(self.chunk_in, 0.0);
- if let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)
- && let Ok(out) = resampler.process(&input, None)
- {
- self.emit_frames(&out.take_data(), &mut emit);
- }
+ let input = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in).map_err(|error| {
+ TranscriptionError::InvalidAudio(format!(
+ "Prepare final audio resampler input: {error}"
+ ))
+ })?;
+ let out = resampler.process(&input, None).map_err(|error| {
+ TranscriptionError::Transcribe(format!("Resample final recorded audio: {error}"))
+ })?;
+ self.emit_frames(&out.take_data(), &mut emit);
}
if !self.pending.is_empty() {
@@ -184,6 +206,7 @@ impl FrameResampler {
emit(&self.pending);
self.pending.clear();
}
+ Ok(())
}
fn emit_frames(&mut self, mut data: &[f32], emit: &mut impl FnMut(&[f32])) {
@@ -208,9 +231,10 @@ mod tests {
use super::*;
#[test]
- fn vad_failure_does_not_return_an_apparently_complete_partial_recording() {
+ fn vad_failure_and_cancellation_do_not_return_partial_recordings() {
struct FailsAfterSpeech {
frames: usize,
+ fail: bool,
}
impl Vad for FailsAfterSpeech {
fn frame_size(&self) -> usize {
@@ -221,7 +245,7 @@ mod tests {
_: &[f32],
) -> std::result::Result<bool, transcribe_rs::TranscribeError> {
self.frames += 1;
- if self.frames == 1 {
+ if self.frames == 1 || !self.fail {
Ok(true)
} else {
Err(transcribe_rs::TranscribeError::Inference(
@@ -231,9 +255,27 @@ mod tests {
}
fn reset(&mut self) {}
}
- let mut vad = FailsAfterSpeech { frames: 0 };
- assert!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad).is_err());
+ let mut vad = FailsAfterSpeech {
+ frames: 0,
+ fail: true,
+ };
+ assert!(
+ matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad, &mut || false), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure"))
+ );
assert_eq!(vad.frames, 2);
+ let mut vad = FailsAfterSpeech {
+ frames: 0,
+ fail: false,
+ };
+ let mut checks = 0;
+ let speech = extract_speech_with_vad(16000, &vec![0.2; 32000], &mut vad, &mut || {
+ checks += 1;
+ checks > 1
+ })
+ .unwrap();
+ assert!(speech.is_empty());
+ assert_eq!(checks, 2);
+ assert_eq!(vad.frames, 16000 / vad.frame_size());
}
fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {
@@ -245,9 +287,9 @@ mod tests {
output.extend_from_slice(frame);
};
for chunk in input.chunks(push_size) {
- resampler.push(chunk, &mut emit);
+ resampler.push(chunk, &mut emit).unwrap();
}
- resampler.finish(emit);
+ resampler.finish(emit).unwrap();
output
}
diff --git a/rust/crates/ensu/src/transcription/graph_cache.rs b/rust/crates/ensu/src/transcription/graph_cache.rs
index 807f739cc6..2443e98427 100644
--- a/rust/crates/ensu/src/transcription/graph_cache.rs
+++ b/rust/crates/ensu/src/transcription/graph_cache.rs
@@ -546,41 +546,41 @@ mod tests {
}
#[test]
- fn serializer_failure_retries_original_and_cleans_staging() {
- let f = Fixture::new();
- let result = f.cache.load(&f.source, "runtime-a", true, |plan| {
- if matches!(plan, LoadPlan::Source { save_to: Some(_) }) {
- f.loads.borrow_mut().push("failed-save");
- Err(io::Error::new(
- io::ErrorKind::StorageFull,
- "injected full disk",
- ))
- } else {
- f.loader(plan)
+ fn failed_cache_load_or_serialization_recovers_from_source() {
+ for cached in [false, true] {
+ let f = Fixture::new();
+ let expected = f.loader(LoadPlan::Source { save_to: None }).unwrap();
+ if cached {
+ assert_eq!(f.load("runtime-a"), expected);
}
- });
- assert!(result.is_ok());
- assert_eq!(f.loads.take(), ["failed-save", "source"]);
- assert!(!f.ready().exists());
- assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 1);
- }
-
- #[test]
- fn rejected_hash_valid_graph_rebuilds_from_source() {
- let f = Fixture::new();
- let expected = f.load("runtime-a");
- let actual = f
- .cache
- .load(&f.source, "runtime-a", true, |plan| {
- if matches!(plan, LoadPlan::Cached { .. }) {
- Err(invalid_cache())
- } else {
- f.loader(plan)
- }
- })
- .unwrap();
- assert_eq!(actual, expected);
- assert_eq!(f.loads.take(), ["build", "build"]);
+ f.loads.take();
+ let mut failed = false;
+ let actual = f
+ .cache
+ .load(&f.source, "runtime-a", true, |plan| match plan {
+ LoadPlan::Cached { .. } if cached => {
+ failed = true;
+ Err(invalid_cache())
+ }
+ LoadPlan::Source { save_to: Some(_) } if !cached => {
+ failed = true;
+ Err(io::Error::new(
+ io::ErrorKind::StorageFull,
+ "injected full disk",
+ ))
+ }
+ _ => f.loader(plan),
+ })
+ .unwrap();
+ assert!(failed);
+ assert_eq!(actual, expected);
+ assert_eq!(f.loads.take(), [if cached { "build" } else { "source" }]);
+ assert_eq!(f.ready().exists(), cached);
+ assert_eq!(
+ fs::read_dir(&f.cache.root).unwrap().count(),
+ if cached { 2 } else { 1 }
+ );
+ }
}
#[test]
diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rs
index 7c281996dc..9d838e9933 100644
--- a/rust/crates/ensu/src/transcription/live.rs
+++ b/rust/crates/ensu/src/transcription/live.rs
@@ -119,14 +119,10 @@ impl LiveTranscription {
|| {
transcriber
.load_model()
- .map_err(|_| VoiceError::ProcessingFailed)
+ .map_err(|error| super::voice_error("Load transcription model", error))
},
|request| transcriber.transcribe_window(&request.samples),
- |pcm| {
- transcriber
- .transcribe(16_000, pcm)
- .map_err(|_| VoiceError::ProcessingFailed)
- },
+ |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),
on_event,
)
}
@@ -142,7 +138,9 @@ impl LiveTranscription {
let _capture = self.start()?;
let mut vad = make_vad()?;
if vad.frame_size() != super::VAD_FRAME_SAMPLES {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Voice activity model has an unsupported frame size".into(),
+ ));
}
let mut normalizer = StreamNormalizer::new(self.queue.format())?;
let mut rolling = RollingRecognizer::default();
@@ -183,9 +181,9 @@ impl LiveTranscription {
let chunk = self.queue.take();
let had_chunk = chunk.is_some();
let mut emit = |frame: &[f32], real: usize| -> Result<()> {
- let voiced = vad
- .is_speech(frame)
- .map_err(|_| VoiceError::ProcessingFailed)?;
+ let voiced = vad.is_speech(frame).map_err(|error| {
+ super::voice_error("Detect speech in live audio", error)
+ })?;
let start = accepted;
accepted += real as u64;
self.emit(
@@ -262,8 +260,10 @@ impl LiveTranscription {
}
}
Err(TryRecvError::Empty) => {}
- Err(TryRecvError::Disconnected) => {
- return Err(VoiceError::ProcessingFailed);
+ Err(error @ TryRecvError::Disconnected) => {
+ return Err(VoiceError::ProcessingFailed(format!(
+ "Receive speech decoder result: {error}"
+ )));
}
}
if prepared && finished && (deferred || rolling.is_settled()) {
@@ -276,9 +276,11 @@ impl LiveTranscription {
&& !deferred
&& let Some(request) = rolling.next_window()?
{
- requests
- .try_send(request)
- .map_err(|_| VoiceError::ProcessingFailed)?;
+ requests.try_send(request).map_err(|error| {
+ VoiceError::ProcessingFailed(format!(
+ "Send speech decoder window: {error}"
+ ))
+ })?;
}
if !had_chunk && !finished {
self.emit(LiveEvent::Idle, on_event)?;
@@ -294,7 +296,9 @@ impl LiveTranscription {
}
drop(requests);
drop(result_rx);
- let exited = decoder.join().map_err(|_| VoiceError::ProcessingFailed);
+ let exited = decoder
+ .join()
+ .map_err(|_| VoiceError::ProcessingFailed("Speech decoder thread panicked".into()));
exited.and(processing)?;
if deferred {
drop(vad);
@@ -492,25 +496,55 @@ mod tests {
}
#[test]
- fn failed_model_initialization_closes_capture_without_recognition() {
- let stream = stream();
- let mut closed = false;
- let result = stream.run_with(
- vad,
- || Err(VoiceError::ProcessingFailed),
- |_| panic!("Failed model must not decode"),
- |_| panic!("Failed model must not transcribe"),
- &mut |event| {
- match event {
- LiveEvent::Ready | LiveEvent::Idle => {}
- LiveEvent::CaptureClosed => closed = true,
- _ => panic!("Failed initialization emitted recognition"),
- }
- Ok(LiveControl::Continue)
- },
- );
- assert_eq!(result, Err(VoiceError::ProcessingFailed));
- assert!(closed);
+ fn model_initialization_failures_preserve_causes_and_close_capture() {
+ use super::super::TranscriptionError;
+ use transcribe_rs::TranscribeError;
+
+ for (error, expected) in [
+ (TranscriptionError::NotDownloaded, VoiceError::NotDownloaded),
+ (
+ TranscriptionError::VadNotDownloaded,
+ VoiceError::VadNotDownloaded,
+ ),
+ (
+ TranscribeError::Io(std::io::ErrorKind::StorageFull.into()).into(),
+ VoiceError::StorageFull,
+ ),
+ (
+ TranscribeError::Io(std::io::Error::other("injected I/O failure")).into(),
+ VoiceError::Io("Load transcription model: injected I/O failure".into()),
+ ),
+ (
+ TranscribeError::Audio("injected audio failure".into()).into(),
+ VoiceError::InvalidAudio("Load transcription model: injected audio failure".into()),
+ ),
+ (
+ TranscribeError::Inference("injected initialization failure".into()).into(),
+ VoiceError::ProcessingFailed(
+ "Load transcription model: inference error: injected initialization failure"
+ .into(),
+ ),
+ ),
+ ] {
+ let stream = stream();
+ let mut closed = false;
+ let result = stream.run_with(
+ vad,
+ || Err(super::super::voice_error("Load transcription model", error)),
+ |_| panic!("Failed model must not decode"),
+ |_| panic!("Failed model must not transcribe"),
+ &mut |event| {
+ match event {
+ LiveEvent::Ready | LiveEvent::Idle => {}
+ LiveEvent::CaptureClosed => closed = true,
+ _ => panic!("Failed initialization emitted recognition"),
+ }
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(result, Err(expected));
+ assert!(closed);
+ }
}
#[test]
diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rs
index 651de4da09..37720cdb56 100644
--- a/rust/crates/ensu/src/transcription/mod.rs
+++ b/rust/crates/ensu/src/transcription/mod.rs
@@ -41,7 +41,12 @@ impl From<std::io::Error> for TranscriptionError {
impl From<transcribe_rs::TranscribeError> for TranscriptionError {
fn from(value: transcribe_rs::TranscribeError) -> Self {
- Self::Transcribe(value.to_string())
+ match value {
+ transcribe_rs::TranscribeError::ModelNotFound(_) => Self::NotDownloaded,
+ transcribe_rs::TranscribeError::Audio(detail) => Self::InvalidAudio(detail),
+ transcribe_rs::TranscribeError::Io(error) => error.into(),
+ error => Self::Transcribe(error.to_string()),
+ }
}
}
@@ -57,4 +62,20 @@ impl From<rubato::ResamplerConstructionError> for TranscriptionError {
}
}
+fn voice_error(operation: &str, error: impl Into<TranscriptionError>) -> crate::voice::VoiceError {
+ use crate::voice::VoiceError;
+ match error.into() {
+ TranscriptionError::NotDownloaded => VoiceError::NotDownloaded,
+ TranscriptionError::VadNotDownloaded => VoiceError::VadNotDownloaded,
+ TranscriptionError::StorageFull => VoiceError::StorageFull,
+ TranscriptionError::InvalidAudio(detail) => {
+ VoiceError::InvalidAudio(format!("{operation}: {detail}"))
+ }
+ TranscriptionError::Transcribe(detail) => {
+ VoiceError::ProcessingFailed(format!("{operation}: {detail}"))
+ }
+ TranscriptionError::Io(error) => VoiceError::Io(format!("{operation}: {error}")),
+ }
+}
+
pub type Result<T> = std::result::Result<T, TranscriptionError>;
diff --git a/rust/crates/ensu/src/transcription/normalize.rs b/rust/crates/ensu/src/transcription/normalize.rs
index 66e50b9d98..bfc0651868 100644
--- a/rust/crates/ensu/src/transcription/normalize.rs
+++ b/rust/crates/ensu/src/transcription/normalize.rs
@@ -32,7 +32,7 @@ impl StreamNormalizer {
WindowFunction::BlackmanHarris2,
FixedSync::Input,
)
- .map_err(|_| VoiceError::InvalidAudio)?,
+ .map_err(|error| super::voice_error("Create audio resampler", error))?,
)
};
let delay = resampler.as_ref().map_or(0, Resampler::output_delay);
@@ -55,7 +55,9 @@ impl StreamNormalizer {
) -> Result<()> {
let frame_bytes = self.format.bytes_per_frame();
if self.finished || !pcm.len().is_multiple_of(frame_bytes) {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Audio must contain complete PCM16 frames before normalization finishes".into(),
+ ));
}
for frame in pcm.chunks_exact(frame_bytes) {
let mono = frame
@@ -89,7 +91,9 @@ impl StreamNormalizer {
let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);
while self.emitted < expected {
let Some(resampler) = &self.resampler else {
- return Err(VoiceError::ProcessingFailed);
+ return Err(VoiceError::ProcessingFailed(
+ "Audio normalization did not produce all input samples".into(),
+ ));
};
self.input.resize(resampler.input_frames_next(), 0.0);
self.process_chunk(emit)?;
@@ -106,11 +110,14 @@ impl StreamNormalizer {
fn process_chunk(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {
let resampler = self.resampler.as_mut().ok_or(VoiceError::InvalidState)?;
- let input = InterleavedSlice::new(&self.input, 1, self.input.len())
- .map_err(|_| VoiceError::InvalidAudio)?;
+ let input = InterleavedSlice::new(&self.input, 1, self.input.len()).map_err(|error| {
+ VoiceError::InvalidAudio(format!("Prepare audio resampler input: {error}"))
+ })?;
let output = resampler
.process(&input, None)
- .map_err(|_| VoiceError::ProcessingFailed)?
+ .map_err(|error| {
+ VoiceError::ProcessingFailed(format!("Resample captured audio: {error}"))
+ })?
.take_data();
self.input.clear();
let skip = self.delay.min(output.len());
diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rs
index 69f7c3f243..133eeaf934 100644
--- a/rust/crates/ensu/src/transcription/recorded.rs
+++ b/rust/crates/ensu/src/transcription/recorded.rs
@@ -7,6 +7,45 @@ use crate::voice::{Result, VoiceError};
const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;
+pub(super) fn transcribe_chunks(
+ mut samples: &[f32],
+ mut decode: impl FnMut(&[f32]) -> super::Result<Option<String>>,
+) -> super::Result<Option<String>> {
+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE, transcript_separator};
+
+ let mut text = String::new();
+ while !samples.is_empty() {
+ let end = if samples.len() <= MAX_WINDOW_SAMPLES {
+ samples.len()
+ } else {
+ let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);
+ let start = limit - 2 * VOICE_SAMPLE_RATE as usize;
+ let offset = samples[start..limit]
+ .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()
+ .0
+ .iter()
+ .enumerate()
+ .map(|(index, frame)| {
+ (
+ index,
+ frame.iter().map(|sample| sample * sample).sum::<f32>(),
+ )
+ })
+ .min_by(|a, b| a.1.total_cmp(&b.1))
+ .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);
+ start + offset
+ };
+ let Some(chunk) = decode(&samples[..end])? else {
+ return Ok(None);
+ };
+ let chunk = chunk.trim();
+ text.push_str(transcript_separator(&text, chunk));
+ text.push_str(chunk);
+ samples = &samples[end..];
+ }
+ Ok(Some(text))
+}
+
#[derive(Default)]
pub(super) struct RecordedPcm(Vec<u8>);
@@ -41,11 +80,7 @@ impl LiveTranscription {
) -> Result<()> {
self.run_recorded_with(
|| transcriber.live_vad(),
- |pcm| {
- transcriber
- .transcribe(16_000, pcm)
- .map_err(|_| VoiceError::ProcessingFailed)
- },
+ |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),
on_event,
)
}
@@ -69,9 +104,9 @@ impl LiveTranscription {
let chunk = self.queue.take();
let had_chunk = chunk.is_some();
let mut emit = |frame: &[f32], real: usize| -> Result<()> {
- let voiced = vad
- .is_speech(frame)
- .map_err(|_| VoiceError::ProcessingFailed)?;
+ let voiced = vad.is_speech(frame).map_err(|error| {
+ super::voice_error("Detect speech in recorded audio", error)
+ })?;
heard_speech |= voiced;
let start = accepted;
accepted += real as u64;
@@ -125,6 +160,42 @@ mod tests {
use crate::voice::CaptureFormat;
use transcribe_rs::vad::EnergyVad;
+ #[test]
+ fn recorded_decoding_bounds_windows_preserves_tails_and_stops_on_cancel() {
+ use crate::voice::MAX_WINDOW_SAMPLES;
+
+ let mut samples = vec![0.25; MAX_WINDOW_SAMPLES * 3 + 1];
+ let quiet = MAX_WINDOW_SAMPLES - 2 * 16000 + 10 * super::super::VAD_FRAME_SAMPLES;
+ samples[quiet..quiet + super::super::VAD_FRAME_SAMPLES].fill(0.0);
+ let mut offset = 0;
+ let mut decoded = 0;
+ let text = transcribe_chunks(&samples, |chunk| {
+ assert!(chunk.len() <= MAX_WINDOW_SAMPLES);
+ assert!(chunk.len() >= 16000);
+ assert_eq!(chunk, &samples[offset..offset + chunk.len()]);
+ if offset == 0 {
+ assert_eq!(chunk.len(), quiet + super::super::VAD_FRAME_SAMPLES);
+ }
+ offset += chunk.len();
+ decoded += 1;
+ Ok(Some(" again ".into()))
+ })
+ .unwrap()
+ .unwrap();
+ assert_eq!(offset, samples.len());
+ assert_eq!(text.split_whitespace().count(), decoded);
+ let mut calls = 0;
+ assert!(
+ transcribe_chunks(&samples, |_| {
+ calls += 1;
+ Ok((calls == 1).then(|| "unfinished".into()))
+ })
+ .unwrap()
+ .is_none()
+ );
+ assert_eq!(calls, 2);
+ }
+
#[test]
fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {
let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);
@@ -138,19 +209,19 @@ mod tests {
}
#[test]
- fn recorded_text_arrives_once_after_close_with_partial_tail() {
- let stream = LiveTranscription::new(CaptureFormat {
- sample_rate: 16000,
- channels: 1,
- })
- .unwrap();
- stream.feed(&[0, 32].repeat(1001)).unwrap();
- stream.finish();
- let mut closed = false;
- let mut commits = Vec::new();
- let mut covered = 0;
- stream
- .run_recorded_with(
+ fn recorded_completion_preserves_the_tail_and_cancellation_suppresses_results() {
+ for cancelled in [false, true] {
+ let stream = LiveTranscription::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap();
+ stream.feed(&[0, 32].repeat(1001)).unwrap();
+ stream.finish();
+ let mut closed = 0;
+ let mut commits = Vec::new();
+ let mut covered = None;
+ let result = stream.run_recorded_with(
|| {
Ok(Box::new(EnergyVad::new(
super::super::VAD_FRAME_SAMPLES,
@@ -158,27 +229,37 @@ mod tests {
)))
},
|pcm| {
- assert_eq!(pcm.len(), 1001 * 2);
assert_eq!(pcm, [0, 32].repeat(1001));
+ if cancelled {
+ stream.cancel();
+ }
Ok("dictated words".into())
},
&mut |event| {
match event {
- LiveEvent::CaptureClosed => closed = true,
+ LiveEvent::CaptureClosed => closed += 1,
LiveEvent::ReplaceTranscript(text) => {
- assert!(closed);
+ assert_eq!(closed, 1);
commits.push(text);
}
LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),
- LiveEvent::Coverage(end) => covered = end,
+ LiveEvent::Coverage(end) => covered = Some(end),
_ => {}
}
Ok(LiveControl::Continue)
},
- )
- .unwrap();
- assert_eq!(commits, ["dictated words"]);
- assert_eq!(covered, 1001);
+ );
+ assert_eq!(closed, 1);
+ if cancelled {
+ assert_eq!(result, Err(VoiceError::Cancelled));
+ assert!(commits.is_empty());
+ assert_eq!(covered, None);
+ } else {
+ result.unwrap();
+ assert_eq!(commits, ["dictated words"]);
+ assert_eq!(covered, Some(1001));
+ }
+ }
}
#[test]
@@ -217,7 +298,6 @@ mod tests {
}
let control = state.apply(event, || stream.finish_if_empty())?;
if idle && state.session.stop_reason() == Some(StopReason::Silence) {
- // A native read can finish after closure, before its stop callback.
assert!(!capture_closed);
assert_eq!(stream.feed(&[0; 3200])?, CaptureFeedResult::Closed);
late_buffer = true;
@@ -236,35 +316,4 @@ mod tests {
assert_eq!(state.text(), "Complete recorded transcript");
assert_eq!(stream.submitted_samples(), (audio.len() / 2) as u64);
}
-
- #[test]
- fn cancellation_during_batch_decode_does_not_publish_text() {
- let stream = LiveTranscription::new(CaptureFormat {
- sample_rate: 16000,
- channels: 1,
- })
- .unwrap();
- stream.feed(&[0, 32].repeat(512)).unwrap();
- stream.finish();
- let result = stream.run_recorded_with(
- || {
- Ok(Box::new(EnergyVad::new(
- super::super::VAD_FRAME_SAMPLES,
- 0.01,
- )))
- },
- |_| {
- stream.cancel();
- Ok("late result".into())
- },
- &mut |event| {
- assert!(!matches!(
- event,
- LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)
- ));
- Ok(LiveControl::Continue)
- },
- );
- assert_eq!(result, Err(VoiceError::Cancelled));
- }
}
diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rs
index 8c88eda0db..cee8f3ae32 100644
--- a/rust/crates/ensu/src/transcription/session.rs
+++ b/rust/crates/ensu/src/transcription/session.rs
@@ -59,7 +59,9 @@ impl DictationState {
}
LiveEvent::Commit { start, end, text } => {
if end <= start || start < self.last_commit_end {
- return Err(voice::VoiceError::InvalidAudio);
+ return Err(voice::VoiceError::InvalidAudio(
+ "Transcript commits must have non-overlapping audio ranges".into(),
+ ));
}
self.last_commit_end = end;
self.session.observe_recognition(!text.trim().is_empty())?;
@@ -149,21 +151,6 @@ mod tests {
);
}
- #[test]
- fn preview_preserves_native_spaces_and_unspaced_scripts() {
- for (stable, partial, expected) in [
- ("明", "倩", "明倩"),
- ("hello", "world", "hello world"),
- ("hello ", "world", "hello world"),
- ] {
- let mut state = DictationState::new(true);
- state.session.begin_capture().unwrap();
- event(&mut state, LiveEvent::ReplaceTranscript(stable.into()));
- event(&mut state, LiveEvent::Hypothesis(partial.into()));
- assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);
- }
- }
-
#[test]
fn recognized_word_boundaries_survive_preview_and_commit() {
fn apply_update(state: &mut DictationState, update: voice::RecognitionUpdate) {
@@ -175,6 +162,15 @@ mod tests {
}
for (words, expected) in [
+ (vec!["明", "倩", "见", "青"], "明倩见青"),
+ (
+ vec!["hello", "world", "again", "today"],
+ "hello world again today",
+ ),
+ (
+ vec!["hello", "world ", "again", "today"],
+ "hello world again today",
+ ),
(
vec!["It", "costs", "$20.", "today."],
"It costs $20. today.",
diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rs
index 16a34ba8ee..a23c23afed 100644
--- a/rust/crates/ensu/src/transcription/transcriber.rs
+++ b/rust/crates/ensu/src/transcription/transcriber.rs
@@ -18,9 +18,19 @@ pub struct Transcriber {
impl Transcriber {
pub(crate) fn live_vad(&self) -> crate::voice::Result<Box<dyn transcribe_rs::vad::Vad>> {
+ let metadata = std::fs::metadata(&self.vad_model_path).map_err(|error| {
+ if error.kind() == std::io::ErrorKind::NotFound {
+ crate::voice::VoiceError::VadNotDownloaded
+ } else {
+ super::voice_error("Read voice activity model", error)
+ }
+ })?;
+ if !metadata.is_file() {
+ return Err(crate::voice::VoiceError::VadNotDownloaded);
+ }
super::SileroV4Vad::new(&self.vad_model_path)
.map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>)
- .map_err(|_| crate::voice::VoiceError::ProcessingFailed)
+ .map_err(|error| super::voice_error("Load voice activity model", error))
}
pub(crate) fn transcribe_window(
@@ -32,7 +42,7 @@ impl Transcriber {
|| samples.len() > MAX_WINDOW_SAMPLES
|| samples.iter().any(|s| !s.is_finite())
{
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio("Recognition requires a nonempty window of finite samples within the duration limit".into()));
}
let mut loaded = self.lock();
let model = ensure_loaded(
@@ -41,7 +51,7 @@ impl Transcriber {
self.graph_cache.as_ref(),
true,
)
- .map_err(|_| VoiceError::ProcessingFailed)?;
+ .map_err(|error| super::voice_error("Load transcription model", error))?;
let result = model
.transcribe_with(
samples,
@@ -50,7 +60,7 @@ impl Transcriber {
..Default::default()
},
)
- .map_err(|_| VoiceError::ProcessingFailed)?;
+ .map_err(|error| super::voice_error("Transcribe live audio", error))?;
result
.segments
.ok_or(VoiceError::UnstableTranscript)?
@@ -110,8 +120,31 @@ impl Transcriber {
}
pub fn transcribe(&self, input_sample_rate: u32, pcm_le: &[u8]) -> Result<String> {
+ self.transcribe_with_cancel(input_sample_rate, pcm_le, || false)
+ .map(Option::unwrap_or_default)
+ }
+
+ pub(crate) fn transcribe_recording(
+ &self,
+ pcm_le: &[u8],
+ is_cancelled: impl FnMut() -> bool,
+ ) -> crate::voice::Result<String> {
+ self.transcribe_with_cancel(16_000, pcm_le, is_cancelled)
+ .map_err(|error| super::voice_error("Transcribe recorded audio", error))?
+ .ok_or(crate::voice::VoiceError::Cancelled)
+ }
+
+ fn transcribe_with_cancel(
+ &self,
+ input_sample_rate: u32,
+ pcm_le: &[u8],
+ mut is_cancelled: impl FnMut() -> bool,
+ ) -> Result<Option<String>> {
+ if is_cancelled() {
+ return Ok(None);
+ }
if pcm_le.is_empty() {
- return Ok(String::new());
+ return Ok(Some(String::new()));
}
if !self.model_dir.is_dir() {
return Err(TranscriptionError::NotDownloaded);
@@ -120,9 +153,17 @@ impl Transcriber {
return Err(TranscriptionError::VadNotDownloaded);
}
- let speech = extract_speech_from_pcm16(&self.vad_model_path, input_sample_rate, pcm_le)?;
+ let speech = extract_speech_from_pcm16(
+ &self.vad_model_path,
+ input_sample_rate,
+ pcm_le,
+ &mut is_cancelled,
+ )?;
+ if is_cancelled() {
+ return Ok(None);
+ }
if speech.is_empty() {
- return Ok(String::new());
+ return Ok(Some(String::new()));
}
let mut loaded = self.lock();
@@ -132,15 +173,19 @@ impl Transcriber {
self.graph_cache.as_ref(),
false,
)?;
- let result = model.transcribe_with(
- &speech,
- &ParakeetParams {
- timestamp_granularity: Some(TimestampGranularity::Segment),
- ..Default::default()
- },
- )?;
-
- Ok(result.text.trim().to_owned())
+ super::recorded::transcribe_chunks(&speech, |samples| {
+ if is_cancelled() {
+ return Ok(None);
+ }
+ let result = model.transcribe_with(
+ samples,
+ &ParakeetParams {
+ timestamp_granularity: Some(TimestampGranularity::Segment),
+ ..Default::default()
+ },
+ )?;
+ Ok((!is_cancelled()).then_some(result.text))
+ })
}
fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {
diff --git a/rust/crates/ensu/src/voice/audio.rs b/rust/crates/ensu/src/voice/audio.rs
index 5ee6a89eb2..9583cf9ad5 100644
--- a/rust/crates/ensu/src/voice/audio.rs
+++ b/rust/crates/ensu/src/voice/audio.rs
@@ -2,13 +2,13 @@ use std::collections::VecDeque;
use super::{Result, VoiceError};
-pub const VOICE_SAMPLE_RATE: u64 = 16_000;
-pub const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;
-pub const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;
-pub const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;
+pub(crate) const VOICE_SAMPLE_RATE: u64 = 16_000;
+pub(crate) const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;
+const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;
+const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;
#[derive(Debug)]
-pub struct AudioRing {
+pub(crate) struct AudioRing {
samples: VecDeque<f32>,
start: u64,
processed: u64,
@@ -27,11 +27,13 @@ impl Default for AudioRing {
impl AudioRing {
pub fn push(&mut self, offset: u64, samples: &[f32]) -> Result<()> {
if offset != self.accepted() || samples.iter().any(|v| !v.is_finite()) {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Invalid audio offset or samples".into(),
+ ));
}
let end = offset
.checked_add(samples.len() as u64)
- .ok_or(VoiceError::InvalidAudio)?;
+ .ok_or_else(|| VoiceError::InvalidAudio("Audio offset overflow".into()))?;
if end - self.processed > MAX_BACKLOG_SAMPLES
|| self.samples.len().saturating_add(samples.len()) > MAX_RING_SAMPLES
{
@@ -47,7 +49,9 @@ impl AudioRing {
|| end < start
|| end - start > MAX_WINDOW_SAMPLES as u64
{
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Audio window is outside the retained range".into(),
+ ));
}
Ok(self
.samples
@@ -64,7 +68,9 @@ impl AudioRing {
|| retain_from < self.start
|| retain_from > processed
{
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Invalid processed audio range".into(),
+ ));
}
self.samples.drain(..(retain_from - self.start) as usize);
self.start = retain_from;
@@ -112,10 +118,19 @@ mod tests {
#[test]
fn invalid_offsets_and_nan_are_rejected_atomically() {
let mut ring = AudioRing::default();
- assert_eq!(ring.push(1, &[0.0]), Err(VoiceError::InvalidAudio));
- assert_eq!(ring.push(0, &[f32::NAN]), Err(VoiceError::InvalidAudio));
+ assert!(matches!(
+ ring.push(1, &[0.0]),
+ Err(VoiceError::InvalidAudio(_))
+ ));
+ assert!(matches!(
+ ring.push(0, &[f32::NAN]),
+ Err(VoiceError::InvalidAudio(_))
+ ));
assert_eq!(ring.accepted(), 0);
ring.push(0, &[0.1, 0.2]).unwrap();
- assert_eq!(ring.acknowledge(3, 0), Err(VoiceError::InvalidAudio));
+ assert!(matches!(
+ ring.acknowledge(3, 0),
+ Err(VoiceError::InvalidAudio(_))
+ ));
}
}
diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rs
index 675782eb50..569ad6340b 100644
--- a/rust/crates/ensu/src/voice/capture.rs
+++ b/rust/crates/ensu/src/voice/capture.rs
@@ -19,12 +19,14 @@ pub struct CaptureFormat {
impl CaptureFormat {
pub fn validate(self) -> Result<Self> {
if !(8_000..=48_000).contains(&self.sample_rate) || !matches!(self.channels, 1 | 2) {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Capture requires 8–48 kHz PCM16 with one or two channels".into(),
+ ));
}
Ok(self)
}
- pub fn bytes_per_frame(self) -> usize {
+ pub(crate) fn bytes_per_frame(self) -> usize {
usize::from(self.channels) * 2
}
}
@@ -36,7 +38,7 @@ struct Pending {
closed: bool,
}
-pub struct CaptureQueue {
+pub(crate) struct CaptureQueue {
format: CaptureFormat,
pending: Mutex<Pending>,
failed: AtomicBool,
@@ -72,7 +74,10 @@ impl CaptureQueue {
|| pcm.len() / frame_bytes > self.format.sample_rate as usize / 5
{
self.failed.store(true, Ordering::Release);
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Capture buffer must contain complete PCM16 frames and at most 200 ms of audio"
+ .into(),
+ ));
}
let owned = pcm.to_vec();
let mut pending = self.lock();
diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rs
index 8f22dadbf5..8357ba0995 100644
--- a/rust/crates/ensu/src/voice/composer.rs
+++ b/rust/crates/ensu/src/voice/composer.rs
@@ -2,7 +2,7 @@ use std::ops::Range;
use super::{Result, VoiceError};
-pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {
+fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {
if start > end {
return Err(VoiceError::InvalidTextRange);
}
@@ -34,8 +34,6 @@ fn left_edge(text: &str) -> Option<char> {
.or_else(|| text.chars().next())
}
-// Recognition already supplies word boundaries. Symbols and punctuation belong to
-// those words and must not suppress their separators, including across commits.
pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {
fn unspaced(c: char) -> bool {
matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)
@@ -50,7 +48,7 @@ pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {
}
}
-pub fn boundary_separator(left: &str, right: &str) -> &'static str {
+fn boundary_separator(left: &str, right: &str) -> &'static str {
fn opening(c: char) -> bool {
matches!(c, 'ΒΏ' | 'Β‘')
|| regex::regex!(r"\A[\p{Ps}\p{Pi}]\z").is_match(c.encode_utf8(&mut [0; 4]))
@@ -103,8 +101,6 @@ pub fn boundary_separator(left: &str, right: &str) -> &'static str {
return "";
}
- // A currency prefix attaches to its amount; a trailing currency symbol is
- // already part of a completed amount and needs a word separator.
if regex::regex!(r"\A\p{Sc}\z").is_match(a.encode_utf8(&mut [0; 4]))
&& !left.rsplit_once(a).is_some_and(|(before, _)| {
before
@@ -173,7 +169,7 @@ pub fn preview_insertion(
#[derive(Debug, Default)]
pub struct ComposerTransfer {
- receipt: Option<(VoiceInsertion, u32)>,
+ applied: bool,
}
impl ComposerTransfer {
@@ -186,12 +182,8 @@ impl ComposerTransfer {
foreground: bool,
composing: bool,
) -> Result<u32> {
- if let Some((applied, receipt)) = &self.receipt {
- return if applied == insertion {
- Ok(*receipt)
- } else {
- Err(VoiceError::RevisionConflict)
- };
+ if self.applied {
+ return Err(VoiceError::RevisionConflict);
}
if !foreground || insertion.anchor.destination != destination {
return Err(VoiceError::RevisionConflict);
@@ -210,7 +202,7 @@ impl ComposerTransfer {
let (rendered, caret) = preview_insertion(text, &anchor, &insertion.text)?;
*text = rendered;
*revision += 1;
- self.receipt = Some((insertion.clone(), caret));
+ self.applied = true;
Ok(caret)
}
}
@@ -232,27 +224,25 @@ mod tests {
}
#[test]
- fn selected_text_is_replaced_once_even_after_lost_acknowledgement() {
+ fn selected_text_is_replaced_once_and_duplicate_transfers_are_rejected() {
let mut owner = ComposerTransfer::default();
let mut text = " meet Friday\n".to_owned();
let mut revision = 7;
let insert = insertion(7, 13, "Saturday");
- let receipt = owner
+ owner
.apply(&insert, "chat", &mut revision, &mut text, true, false)
.unwrap();
assert_eq!(text, " meet Saturday\n");
assert_eq!(
- owner
- .apply(
- &insert,
- "different-chat",
- &mut revision,
- &mut text,
- false,
- false
- )
- .unwrap(),
- receipt
+ owner.apply(
+ &insert,
+ "different-chat",
+ &mut revision,
+ &mut text,
+ false,
+ false
+ ),
+ Err(VoiceError::RevisionConflict)
);
assert_eq!(revision, 8);
assert_eq!(text, " meet Saturday\n");
@@ -287,9 +277,10 @@ mod tests {
.apply(&insert, "chat", &mut revision, &mut text, true, false)
.unwrap();
assert_eq!(text, "My edited draft hello");
- owner
- .apply(&insert, "chat", &mut revision, &mut text, true, false)
- .unwrap();
+ assert_eq!(
+ owner.apply(&insert, "chat", &mut revision, &mut text, true, false),
+ Err(VoiceError::RevisionConflict)
+ );
assert_eq!(text, "My edited draft hello");
assert_eq!(revision, 9);
}
@@ -379,7 +370,6 @@ mod tests {
caret
);
assert_eq!(text, expected);
- // The same boundary can occur after the inserted text, too.
let anchor = insertion(0, 0, draft).anchor;
assert_eq!(
preview_insertion(dictated, &anchor, draft).unwrap().0,
diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rs
index dc70c90c0c..2d466e5a66 100644
--- a/rust/crates/ensu/src/voice/mod.rs
+++ b/rust/crates/ensu/src/voice/mod.rs
@@ -4,24 +4,36 @@ mod composer;
mod rolling;
mod session;
-pub use audio::*;
-pub use capture::*;
-pub use composer::*;
-pub use rolling::*;
-pub use session::*;
+pub(crate) use audio::{AudioRing, MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};
+pub(crate) use capture::CaptureQueue;
+pub use capture::{CaptureFeedResult, CaptureFormat};
+pub(crate) use composer::transcript_separator;
+pub use composer::{ComposerAnchor, ComposerTransfer, VoiceInsertion, preview_insertion};
+#[cfg(test)]
+pub(crate) use rolling::RecognitionUpdate;
+pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer};
+pub use session::{StopReason, VoicePhase, VoiceSession};
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum VoiceError {
- #[error("Audio continuity or format is invalid")]
- InvalidAudio,
+ #[error("{0}")]
+ InvalidAudio(String),
+ #[error("Transcription model is not downloaded")]
+ NotDownloaded,
+ #[error("Voice activity model is not downloaded")]
+ VadNotDownloaded,
+ #[error("not enough storage space")]
+ StorageFull,
+ #[error("{0}")]
+ Io(String),
#[error("Audio backlog limit reached")]
BacklogFull,
#[error("Voice recording was cancelled")]
Cancelled,
#[error("Speech alignment could not be preserved")]
UnstableTranscript,
- #[error("Speech processing failed")]
- ProcessingFailed,
+ #[error("{0}")]
+ ProcessingFailed(String),
#[error("Voice event belongs to an inactive recording")]
StaleEvent,
#[error("Voice operation is not valid in the current state")]
diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rs
index cbd9e9050e..25d06ae8bc 100644
--- a/rust/crates/ensu/src/voice/rolling.rs
+++ b/rust/crates/ensu/src/voice/rolling.rs
@@ -13,13 +13,13 @@ const REQUIRED_OBSERVATIONS: u8 = 3;
const RIGHT_CONTEXT_WORDS: usize = 2;
#[derive(Clone, Debug)]
-pub struct AlignedWord {
+pub(crate) struct AlignedWord {
pub text: String,
pub start: u64,
pub end: u64,
}
-pub struct DecodeWindow {
+pub(crate) struct DecodeWindow {
pub id: u64,
pub start: u64,
pub end: u64,
@@ -28,7 +28,7 @@ pub struct DecodeWindow {
}
#[derive(Debug)]
-pub struct RecognitionUpdate {
+pub(crate) struct RecognitionUpdate {
pub commit: Option<(u64, u64, String)>,
pub hypothesis: String,
pub covered_through: u64,
@@ -41,7 +41,7 @@ struct Region {
}
#[derive(Default)]
-pub struct RollingRecognizer {
+pub(crate) struct RollingRecognizer {
ring: AudioRing,
regions: VecDeque<Region>,
previous: Vec<AlignedWord>,
@@ -56,7 +56,7 @@ pub struct RollingRecognizer {
impl RollingRecognizer {
pub fn feed(&mut self, samples: &[f32], voiced: bool) -> Result<()> {
if samples.is_empty() {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio("Audio buffer is empty".into()));
}
let start = self.ring.accepted();
self.ring.push(start, samples)?;
diff --git a/rust/crates/ensu/src/voice/session.rs b/rust/crates/ensu/src/voice/session.rs
index 613015d145..a15b3a9024 100644
--- a/rust/crates/ensu/src/voice/session.rs
+++ b/rust/crates/ensu/src/voice/session.rs
@@ -103,7 +103,9 @@ impl VoiceSession {
|| end <= start
|| voiced_end.is_some_and(|v| v <= start || v > end)
{
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Audio must be continuous and contain a valid speech range".into(),
+ ));
}
self.accepted = end;
if let Some(voice) = voiced_end {
@@ -182,7 +184,9 @@ impl VoiceSession {
return Err(VoiceError::InvalidState);
}
if end < self.recognized || end > self.accepted {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Recognition coverage is outside the accepted audio range".into(),
+ ));
}
self.recognized = end;
Ok(())
@@ -192,7 +196,9 @@ impl VoiceSession {
return Err(VoiceError::InvalidState);
}
if start != self.accepted || end <= start {
- return Err(VoiceError::InvalidAudio);
+ return Err(VoiceError::InvalidAudio(
+ "Drained audio must continue the accepted audio range".into(),
+ ));
}
self.accepted = end;
Ok(())
@@ -255,38 +261,40 @@ mod tests {
}
#[test]
- fn live_grace_waits_for_text_or_fifteen_seconds() {
- let mut session = VoiceSession::default();
- session.begin_capture().unwrap();
- session.observe_audio(0, 32000, Some(24000)).unwrap();
- session.observe_recognition(false).unwrap();
- session.observe_audio(32000, 80000, None).unwrap();
- assert_eq!(session.phase(), VoicePhase::Listening);
- session.observe_audio(80000, 13 * 16000, None).unwrap();
- assert!(!session.silence_elapsed());
- assert_eq!(session.silence_deadline(), Some(15 * 16000));
- session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();
- assert!(session.silence_elapsed());
- }
-
- #[test]
- fn recognition_and_later_speech_recompute_the_current_deadline() {
- let mut session = VoiceSession::default();
- session.begin_capture().unwrap();
- session.observe_audio(0, 16000, Some(16000)).unwrap();
- session.observe_audio(16000, 13 * 16000, None).unwrap();
- assert!(!session.silence_elapsed());
- session.observe_recognition(true).unwrap();
- assert!(session.silence_elapsed());
- session
- .observe_audio(13 * 16000, 14 * 16000, Some(14 * 16000))
- .unwrap();
- assert!(!session.silence_elapsed());
- assert_eq!(session.phase(), VoicePhase::Listening);
- session.observe_audio(14 * 16000, 16 * 16000, None).unwrap();
- assert!(!session.silence_elapsed());
- session.observe_audio(16 * 16000, 17 * 16000, None).unwrap();
- assert!(session.silence_elapsed());
+ fn live_grace_ends_with_text_or_timeout_and_later_speech_resets_silence() {
+ for has_text in [false, true] {
+ let mut session = VoiceSession::default();
+ session.begin_capture().unwrap();
+ session.observe_audio(0, 32000, Some(24000)).unwrap();
+ session.observe_recognition(false).unwrap();
+ session.observe_audio(32000, 80000, None).unwrap();
+ assert_eq!(session.phase(), VoicePhase::Listening);
+ session.observe_audio(80000, 13 * 16000, None).unwrap();
+ assert!(!session.silence_elapsed());
+ assert_eq!(session.silence_deadline(), Some(15 * 16000));
+ let resumed_at = if has_text {
+ session.observe_recognition(true).unwrap();
+ 13 * 16000
+ } else {
+ session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();
+ 15 * 16000
+ };
+ assert!(session.silence_elapsed());
+ let voice_end = resumed_at + 16000;
+ session
+ .observe_audio(resumed_at, voice_end, Some(voice_end))
+ .unwrap();
+ assert!(!session.silence_elapsed());
+ assert_eq!(session.phase(), VoicePhase::Listening);
+ session
+ .observe_audio(voice_end, voice_end + 32000, None)
+ .unwrap();
+ assert!(!session.silence_elapsed());
+ session
+ .observe_audio(voice_end + 32000, voice_end + 48000, None)
+ .unwrap();
+ assert!(session.silence_elapsed());
+ }
}
#[test]