Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Add inline mobile dictation with faster startup

setalp79673334d053
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index e692e7d94d..f5822f52d1 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -1,13 +1,16 @@
package io.ente.ensu
import android.content.Context
+import android.icu.text.BreakIterator
import io.ente.ensu.assets.AssetStore
import io.ente.ensu.bindings.ConfigDefaults
+import io.ente.ensu.bindings.DictationInsertion
import io.ente.ensu.chat.Attachment
import io.ente.ensu.chat.AttachmentStoreActions
import io.ente.ensu.chat.ChatMessage
import io.ente.ensu.chat.ChatRepository
import io.ente.ensu.chat.ChatStoreActions
+import io.ente.ensu.chat.VoiceTranscriptionController
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
import io.ente.ensu.device.ChatDeviceCapability
import io.ente.ensu.knowledge.KnowledgeProvider
@@ -21,6 +24,8 @@ import io.ente.ensu.logging.LogLevel
import io.ente.ensu.notes.NotesStore
import io.ente.ensu.settings.DeveloperSettingsState
import io.ente.ensu.settings.SessionPreferencesDataStore
+import java.util.Locale
+import java.util.UUID
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
@@ -84,6 +89,60 @@ class AppStore(
awaitKnowledgeReady = knowledgeStore::awaitReady,
)
+ private var composerRevision = 0uL
+ private var emptyComposerId = UUID.randomUUID().toString()
+ internal val voiceController: VoiceTranscriptionController =
+ VoiceTranscriptionController(
+ context.applicationContext,
+ assetStore,
+ llmProvider,
+ logRepository,
+ ::trackVoiceInput,
+ ) { insertion, transcript ->
+ val chat = _state.value.chat
+ runCatching {
+ val result =
+ insertion.apply(
+ chat.currentSessionId ?: emptyComposerId,
+ composerRevision,
+ chat.messageText,
+ transcript,
+ true,
+ false,
+ )
+ chatActions.updateMessageText(result.text)
+ composerRevision = result.revision
+ voiceController.composerCaret = result.caretUtf16.toInt()
+ true
+ }
+ .getOrDefault(false)
+ }
+
+ internal fun openVoiceInput(
+ text: String,
+ selectionStart: Int,
+ selectionEnd: Int,
+ isComposing: Boolean,
+ ) {
+ val chat = _state.value.chat
+ if (isComposing || chat.isGenerating || chat.isDownloading || chat.editingMessageId != null)
+ return
+ if (chat.messageText != text || !voiceController.canOpen) return
+ if (selectionStart !in 0..selectionEnd || selectionEnd > text.length) return
+ val characters = BreakIterator.getCharacterInstance(Locale.ROOT).apply { setText(text) }
+ if (!characters.isBoundary(selectionStart) || !characters.isBoundary(selectionEnd)) return
+ voiceController.open(
+ DictationInsertion(
+ chat.currentSessionId ?: emptyComposerId,
+ composerRevision,
+ selectionStart.toUInt(),
+ selectionEnd.toUInt(),
+ )
+ )
+ }
+
+ fun dispose() = voiceController.dispose()
+
fun bootstrap(scope: CoroutineScope) {
notesStore.bootstrap(scope)
chatActions.setScope(scope)
@@ -100,11 +159,15 @@ class AppStore(
modelSettingsActions.setChatActive(active)
}
- fun setAppForeground(active: Boolean) = modelSettingsActions.setAppForeground(active)
+ fun setAppForeground(active: Boolean) {
+ voiceController.setForeground(active)
+ modelSettingsActions.setAppForeground(active)
+ }
fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)
fun handleMemoryPressure() {
+ voiceController.interrupt("Voice input paused because memory is low.")
chatActions.cancelSessionSummary()
modelSettingsActions.handleMemoryPressure()
}
@@ -173,21 +236,43 @@ class AppStore(
fun createNewSession(): String = chatActions.createNewSession()
- fun startNewSessionDraft() = chatActions.startNewSessionDraft()
+ fun startNewSessionDraft() {
+ voiceController.discard()
+ emptyComposerId = UUID.randomUUID().toString()
+ composerRevision++
+ chatActions.startNewSessionDraft()
+ }
- fun selectSession(sessionId: String) = chatActions.selectSession(sessionId)
+ fun selectSession(sessionId: String) {
+ voiceController.discard()
+ composerRevision++
+ chatActions.selectSession(sessionId)
+ }
- fun deleteSession(sessionId: String) = chatActions.deleteSession(sessionId)
+ fun deleteSession(sessionId: String) {
+ if (sessionId == _state.value.chat.currentSessionId) voiceController.discard()
+ chatActions.deleteSession(sessionId)
+ }
fun persistSelectedSession(scope: CoroutineScope, sessionId: String?) =
chatActions.persistSelectedSession(scope, sessionId)
- fun updateMessageText(value: String) = chatActions.updateMessageText(value)
+ fun updateMessageText(value: String) {
+ if (value != _state.value.chat.messageText) {
+ composerRevision++
+ voiceController.composerCaret = null
+ }
+ chatActions.updateMessageText(value)
+ }
fun updateBranchSelection(messageId: String, selectedIndex: Int) =
chatActions.updateBranchSelection(messageId, selectedIndex)
- fun beginEditing(messageId: String) = chatActions.beginEditing(messageId)
+ fun beginEditing(messageId: String) {
+ voiceController.discard()
+ composerRevision++
+ chatActions.beginEditing(messageId)
+ }
fun cancelEditing() = chatActions.cancelEditing()
@@ -232,7 +317,12 @@ class AppStore(
fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)
- fun sendMessage() = chatActions.sendMessage()
+ fun sendMessage() {
+ if (voiceController.active) return
+ voiceController.composerCaret = null
+ composerRevision++
+ chatActions.sendMessage()
+ }
fun confirmOverflowTrim() = chatActions.confirmOverflowTrim()
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
index 6a57f9f65c..f775249cb9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
@@ -96,6 +96,11 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
}
}
+ override fun onCleared() {
+ store.dispose()
+ super.onCleared()
+ }
+
@Suppress("DEPRECATION")
private fun getAppVersion(application: Application): String {
val packageManager = application.packageManager
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
index b1d7b33041..40576932d7 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
@@ -1,6 +1,7 @@
package io.ente.ensu
import android.app.Application
+import android.os.Build
import io.ente.ensu.assets.AssetStore
import io.ente.ensu.bindings.RustLogLevel
import io.ente.ensu.bindings.RustLogSink
@@ -18,9 +19,11 @@ class EnsuApplication : Application() {
val transcriber by lazy {
val store = assetStore
val transcription = transcriptionModelAsset()
- Transcriber(
+ Transcriber.withCache(
store.assetDir(transcription).absolutePath,
store.voiceActivityModelPath().absolutePath,
+ cacheDir.absolutePath,
+ "${Build.FINGERPRINT}|${Build.HARDWARE}|${Build.SUPPORTED_ABIS.joinToString()}",
)
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index a4c47757e0..b306bf813e 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -111,15 +111,21 @@ internal fun HomeNavigation(
popExitTransition = { backExit() },
) {
ChatView(
- onVoiceInputJob = store::trackVoiceInput,
+ voiceController = store.voiceController,
+ onOpenVoiceInput = { value ->
+ store.openVoiceInput(
+ value.text,
+ value.selection.min,
+ value.selection.max,
+ value.composition != null,
+ )
+ },
onOpenPacks = { navController.navigate(HomeRoute.Knowledge) },
onOpenNotes = { navController.navigate(HomeRoute.NotesSetup) },
hasSeenSourceIntroduction = store.hasSeenSourceIntroduction(),
onCompleteSourceIntroduction = store::completeSourceIntroduction,
onRetrySources = store::retrySourceInventory,
chatState = appState.chat,
- assetStore = store.assetStore,
- llmProvider = store.llmProvider,
isDrawerOpen = drawerState.currentValue == DrawerValue.Open,
onMessageChange = store::updateMessageText,
onSend = store::sendMessage,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
index b5f9cf9162..599d384af5 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeView.kt
@@ -214,15 +214,16 @@ fun HomeView(
} else if (event == Lifecycle.Event.ON_STOP) {
latestStore.notesStore.setForeground(false)
latestStore.setChatActive(false)
- latestStore.setAppForeground(false)
+ if ((context as? android.app.Activity)?.isChangingConfigurations != true)
+ latestStore.setAppForeground(false)
}
}
latestStore.notesStore.setForeground(
lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)
)
- latestStore.setAppForeground(
- lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)
- )
+ if (lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)) {
+ latestStore.setAppForeground(true)
+ }
lifecycleOwner.lifecycle.addObserver(observer)
latestStore.setChatActive(
isChatRoute &&
@@ -230,12 +231,14 @@ fun HomeView(
)
onDispose {
latestStore.setChatActive(false)
- latestStore.setAppForeground(false)
+ if ((context as? android.app.Activity)?.isChangingConfigurations != true)
+ latestStore.setAppForeground(false)
lifecycleOwner.lifecycle.removeObserver(observer)
}
}
LaunchedEffect(isChatRoute, lifecycleOwner) {
+ if (!isChatRoute) latestStore.voiceController.interrupt()
latestStore.setChatActive(
isChatRoute &&
lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
index b54de38b48..9c3938b402 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
@@ -21,8 +21,6 @@ import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.text.BasicTextField
import androidx.compose.foundation.text.KeyboardOptions
import androidx.compose.foundation.verticalScroll
-import androidx.compose.material.icons.Icons
-import androidx.compose.material.icons.rounded.ErrorOutline
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
@@ -76,8 +74,9 @@ internal fun MessageInput(
onAttachmentSelected: (AttachmentType) -> Unit,
onRemoveAttachment: (Attachment) -> Unit,
onCancelEdit: () -> Unit,
- voiceInputState: VoiceInputState,
- onVoiceInput: () -> Unit,
+ voiceController: VoiceTranscriptionController,
+ onVoiceInput: (TextFieldValue) -> Unit,
+ voiceCaret: Int?,
focusRequestId: Int,
) {
val haptic = rememberHaptics()
@@ -94,12 +93,15 @@ internal fun MessageInput(
)
}
- LaunchedEffect(messageText) {
- if (messageText != fieldValue.text) {
+ LaunchedEffect(messageText, voiceCaret, voiceController.active) {
+ if (messageText != fieldValue.text || voiceCaret != null) {
fieldValue =
TextFieldValue(
text = messageText,
- selection = TextRange(messageText.length),
+ selection =
+ TextRange(
+ voiceCaret?.coerceIn(0, messageText.length) ?: messageText.length
+ ),
)
}
}
@@ -196,50 +198,7 @@ internal fun MessageInput(
}
}
- voiceInputState.statusText()?.let { status ->
- Row(
- modifier = Modifier.padding(horizontal = EnsuSpacing.sm.dp),
- verticalAlignment = Alignment.CenterVertically,
- ) {
- when (voiceInputState) {
- is VoiceInputState.Downloading,
- VoiceInputState.Transcribing -> {
- CircularProgressIndicator(
- modifier = Modifier.size(16.dp),
- strokeWidth = 2.dp,
- )
- }
- VoiceInputState.Recording -> {
- Icon(
- painter = painterResource(HugeIcons.Mic02Icon),
- contentDescription = null,
- modifier = Modifier.size(16.dp),
- tint = EnsuColor.stopButton,
- )
- }
- is VoiceInputState.Error -> {
- Icon(
- imageVector = Icons.Rounded.ErrorOutline,
- contentDescription = null,
- modifier = Modifier.size(16.dp),
- tint = EnsuColor.stopButton,
- )
- }
- VoiceInputState.Idle -> Unit
- }
- Spacer(modifier = Modifier.width(EnsuSpacing.sm.dp))
- Text(
- text = status,
- style = EnsuTypography.small,
- color =
- if (voiceInputState is VoiceInputState.Error) {
- EnsuColor.stopButton
- } else {
- EnsuColor.textMuted()
- },
- )
- }
- }
+ VoiceInputControls(voiceController)
Box(
modifier =
@@ -259,7 +218,7 @@ internal fun MessageInput(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
) {
- if (supportsImageAttachments) {
+ if (supportsImageAttachments && !voiceController.active) {
IconButton(
onClick = {
haptic.perform(HapticFeedbackType.TextHandleMove)
@@ -287,13 +246,14 @@ internal fun MessageInput(
value = fieldValue,
onValueChange = { newValue ->
fieldValue = newValue
- onMessageChange(newValue.text)
+ if (!voiceController.active) onMessageChange(newValue.text)
},
modifier = Modifier.weight(1f).focusRequester(focusRequester),
textStyle = EnsuTypography.message.copy(color = EnsuColor.textPrimary()),
visualTransformation = LongMessageVisualTransformation,
minLines = 1,
- maxLines = 5,
+ maxLines = if (voiceController.active) 8 else 5,
+ readOnly = voiceController.active,
keyboardOptions =
KeyboardOptions(capitalization = KeyboardCapitalization.Sentences),
cursorBrush = SolidColor(EnsuColor.accent()),
@@ -314,53 +274,20 @@ internal fun MessageInput(
}
if (editingMessage == null) {
- val isVoiceBusy =
- voiceInputState is VoiceInputState.Downloading ||
- voiceInputState is VoiceInputState.Transcribing
- val canUseVoice =
- voiceInputState.isRecording ||
- (!isGenerating && !isDownloading && !isVoiceBusy)
-
- IconButton(
- onClick = {
- haptic.perform(
- if (voiceInputState.isRecording) {
- HapticFeedbackType.LongPress
- } else {
- HapticFeedbackType.TextHandleMove
- }
- )
- onVoiceInput()
+ VoiceInputButton(
+ controller = voiceController,
+ enabled = !isGenerating && !isDownloading && fieldValue.composition == null,
+ onOpen = {
+ haptic.perform(HapticFeedbackType.TextHandleMove)
+ onVoiceInput(fieldValue)
},
- enabled = canUseVoice,
- modifier = Modifier.size(36.dp),
- ) {
- if (isVoiceBusy) {
- CircularProgressIndicator(
- modifier = Modifier.size(16.dp),
- strokeWidth = 2.dp,
- )
- } else if (voiceInputState.isRecording) {
- Icon(
- painter = painterResource(HugeIcons.StopIcon),
- contentDescription = "Stop dictation",
- modifier = Modifier.size(composerIconSize),
- tint = EnsuColor.sendButton,
- )
- } else {
- Icon(
- painter = painterResource(HugeIcons.Mic02Icon),
- contentDescription = "Dictate",
- modifier = Modifier.size(composerIconSize),
- )
- }
- }
+ )
}
val canSend =
sourcesReady &&
(messageText.isNotBlank() || attachments.isNotEmpty()) &&
- !voiceInputState.blocksSend &&
+ !voiceController.active &&
!hasUnsupportedImages
IconButton(
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 89cddf3df3..6b0d17dca9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -947,6 +947,9 @@ internal class ChatStoreActions(
systemPrompt = normalSystemPrompt,
)
+ if (normalHistorySelection.currentInputTooLarge) {
+ throw ConversationException.CurrentInputTooLarge()
+ }
if (
normalHistorySelection.wasTrimmed &&
overflowBypassMessageId != userMessage.id
@@ -1086,6 +1089,18 @@ internal class ChatStoreActions(
throw err
} catch (_: LlmException.Cancelled) {
interrupted = true
+ } catch (_: ConversationException.CurrentInputTooLarge) {
+ if (isActive() && !stopRequested) {
+ state.update {
+ it.copy(
+ chat =
+ it.chat.copy(
+ transientAssistantError = CURRENT_INPUT_TOO_LARGE_MESSAGE,
+ transientAssistantParentId = userMessage.id,
+ )
+ )
+ }
+ }
} catch (err: ModelMemoryUnavailable) {
if (isActive() && !stopRequested) {
state.update {
@@ -1326,9 +1341,12 @@ internal class ChatStoreActions(
chat =
it.chat.copy(
transientAssistantError =
- (error as? ConversationException.Other)?.detail
- ?: error.message
- ?: "Unable to prepare conversation",
+ if (error is ConversationException.CurrentInputTooLarge)
+ CURRENT_INPUT_TOO_LARGE_MESSAGE
+ else
+ (error as? ConversationException.Other)?.detail
+ ?: error.message
+ ?: "Unable to prepare conversation",
transientAssistantParentId = userMessage.id,
)
)
@@ -1843,6 +1861,7 @@ internal class ChatStoreActions(
val inputTokens: Int,
val inputBudget: Int,
val wasTrimmed: Boolean,
+ val currentInputTooLarge: Boolean,
)
private data class GenerationLimits(
@@ -1914,6 +1933,7 @@ internal class ChatStoreActions(
inputTokens,
inputBudget,
inputTokens > inputBudget,
+ systemTokens + promptTokens > inputBudget,
)
}
@@ -1944,7 +1964,13 @@ internal class ChatStoreActions(
)
}
- return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)
+ return HistorySelection(
+ selected,
+ inputTokens,
+ inputBudget,
+ inputTokens > inputBudget,
+ false,
+ )
}
private fun resolveGenerationLimits(contextLength: Int): GenerationLimits {
@@ -2091,6 +2117,9 @@ internal class ChatStoreActions(
}
companion object {
+ private const val CURRENT_INPUT_TOO_LARGE_MESSAGE =
+ "This message is too long for the model, even without earlier messages. " +
+ "Shorten it or split it into smaller messages. Your full text is still available to edit or copy."
private const val MEDIA_MARKER = "<__media__>"
private const val OVERFLOW_SAFETY_TOKENS = 256
private const val IMAGE_TOKEN_ESTIMATE = 768
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index 31f9beb63b..9bce45e391 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -1,9 +1,5 @@
package io.ente.ensu.chat
-import android.Manifest
-import android.content.pm.PackageManager
-import androidx.activity.compose.rememberLauncherForActivityResult
-import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.AnimatedContent
import androidx.compose.animation.core.FastOutSlowInEasing
import androidx.compose.animation.core.tween
@@ -36,28 +32,22 @@ import androidx.compose.runtime.key
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
-import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.layout.onGloballyPositioned
-import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.unit.dp
-import androidx.core.content.ContextCompat
-import io.ente.ensu.assets.AssetStore
import io.ente.ensu.designsystem.EnsuColor
import io.ente.ensu.designsystem.EnsuSpacing
import io.ente.ensu.device.ChatDeviceCapability
-import io.ente.ensu.llm.LlmProvider
-import kotlinx.coroutines.Job
@Composable
-fun ChatView(
+internal fun ChatView(
chatState: ChatState,
- assetStore: AssetStore,
- llmProvider: LlmProvider,
+ voiceController: VoiceTranscriptionController,
+ onOpenVoiceInput: (androidx.compose.ui.text.input.TextFieldValue) -> Unit,
isDrawerOpen: Boolean,
onMessageChange: (String) -> Unit,
onSend: () -> Unit,
@@ -74,7 +64,6 @@ fun ChatView(
onDismissUnsupportedDeviceDialog: () -> Unit,
onOverflowTrim: () -> Unit,
onOverflowCancel: () -> Unit,
- onVoiceInputJob: (Job) -> Unit,
onOpenPacks: () -> Unit,
onOpenNotes: () -> Unit,
hasSeenSourceIntroduction: Boolean,
@@ -82,28 +71,13 @@ fun ChatView(
onRetrySources: () -> Unit,
) {
val density = LocalDensity.current
- val context = LocalContext.current
var inputBarHeightDp by remember { mutableStateOf(0.dp) }
- val latestMessageText by rememberUpdatedState(chatState.messageText)
- val latestOnMessageChange by rememberUpdatedState(onMessageChange)
val sessionKey = chatState.currentSessionId ?: "new-session"
var introductionSeen by
remember(hasSeenSourceIntroduction) { mutableStateOf(hasSeenSourceIntroduction) }
- val latestSessionKey by rememberUpdatedState(sessionKey)
val unsupportedCapability =
chatState.deviceCapability as? ChatDeviceCapability.UnsupportedLowMemory
val isChatUnsupported = unsupportedCapability != null
- var pendingVoiceSessionKey by remember { mutableStateOf<String?>(null) }
- val voiceController =
- rememberVoiceTranscriptionController(
- assetStore = assetStore,
- llmProvider = llmProvider,
- onVoiceInputJob = onVoiceInputJob,
- onTranscript = { transcript ->
- latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))
- },
- )
-
val editingMessage by
remember(chatState.editingMessageId, chatState.messages) {
derivedStateOf {
@@ -112,30 +86,6 @@ fun ChatView(
}
}
}
- val canStartVoiceInput =
- !chatState.isGenerating &&
- !chatState.isDownloading &&
- chatState.isModelDownloaded &&
- !isChatUnsupported &&
- editingMessage == null
- val latestCanStartVoiceInput by rememberUpdatedState(canStartVoiceInput)
-
- val microphonePermissionLauncher =
- rememberLauncherForActivityResult(contract = ActivityResultContracts.RequestPermission()) {
- granted ->
- val requestedSessionKey = pendingVoiceSessionKey
- pendingVoiceSessionKey = null
- if (granted) {
- if (requestedSessionKey != null) {
- voiceController.startRecording {
- latestSessionKey == requestedSessionKey && latestCanStartVoiceInput
- }
- }
- } else {
- voiceController.onPermissionDenied()
- }
- }
-
val showDownloadOnboarding by
remember(
chatState.isModelDownloaded,
@@ -202,11 +152,6 @@ fun ChatView(
}
}
- LaunchedEffect(sessionKey) {
- pendingVoiceSessionKey = null
- voiceController.cancelActiveVoiceInput()
- }
-
if (showIntroduction) {
SourceIntroductionSheet(
onDismiss = {
@@ -290,7 +235,8 @@ fun ChatView(
inputBarHeightDp = with(density) { coords.size.height.toDp() }
}
)
- } else if (chatState.isModelStateKnown && !showDownloadOnboarding) {
+ } else if (chatState.isModelStateKnown) {
+ val voicePreview = voiceController.composerPreview(chatState.messageText)
MessageInput(
modifier =
Modifier.fillMaxWidth()
@@ -299,7 +245,7 @@ fun ChatView(
.onGloballyPositioned { coords ->
inputBarHeightDp = with(density) { coords.size.height.toDp() }
},
- messageText = chatState.messageText,
+ messageText = voicePreview?.text ?: chatState.messageText,
attachments = chatState.attachments,
supportsImageAttachments = chatState.supportsImageAttachments,
editingMessage = editingMessage,
@@ -318,30 +264,12 @@ fun ChatView(
onAttachmentSelected = onAttachmentSelected,
onRemoveAttachment = onRemoveAttachment,
onCancelEdit = onCancelEdit,
- voiceInputState = voiceController.state,
- onVoiceInput = {
- if (voiceController.state.isRecording) {
- voiceController.stopAndTranscribe()
- } else {
- val hasMicrophonePermission =
- ContextCompat.checkSelfPermission(
- context,
- Manifest.permission.RECORD_AUDIO,
- ) == PackageManager.PERMISSION_GRANTED
- if (canStartVoiceInput && hasMicrophonePermission) {
- val requestedSessionKey = sessionKey
- voiceController.startRecording {
- latestSessionKey == requestedSessionKey &&
- latestCanStartVoiceInput
- }
- } else if (canStartVoiceInput) {
- pendingVoiceSessionKey = sessionKey
- microphonePermissionLauncher.launch(
- Manifest.permission.RECORD_AUDIO
- )
- }
- }
+ voiceController = voiceController,
+ onVoiceInput = { value ->
+ onOpenVoiceInput(value)
+ focusManager.clearFocus()
},
+ voiceCaret = voicePreview?.caretUtf16?.toInt() ?: voiceController.composerCaret,
focusRequestId = focusRequestId,
)
}
@@ -398,15 +326,3 @@ fun ChatView(
}
}
}
-
-private fun appendVoiceTranscript(currentText: String, transcript: String): String {
- val cleaned = transcript.trim()
- if (cleaned.isBlank()) {
- return currentText
- }
- val trimmedDraft = currentText.trimEnd()
- if (trimmedDraft.isBlank()) {
- return cleaned
- }
- return "$trimmedDraft $cleaned"
-}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index 1aa2754e29..52b58d1407 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -1,51 +1,55 @@
package io.ente.ensu.chat
import android.annotation.SuppressLint
+import android.content.Context
+import android.content.pm.ApplicationInfo
import android.media.AudioFormat
+import android.media.AudioManager
import android.media.AudioRecord
import android.media.MediaRecorder
-import android.util.Log
-import androidx.compose.runtime.Composable
-import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
-import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
-import androidx.lifecycle.Lifecycle
-import androidx.lifecycle.LifecycleEventObserver
-import androidx.lifecycle.compose.LocalLifecycleOwner
import io.ente.ensu.assets.AssetStore
-import io.ente.ensu.bindings.AssetDownloadException
-import io.ente.ensu.bindings.LlmException
-import io.ente.ensu.bindings.TranscriptionException
+import io.ente.ensu.bindings.DictationException
+import io.ente.ensu.bindings.DictationInsertion
+import io.ente.ensu.bindings.DictationPhase
+import io.ente.ensu.bindings.DictationSession
+import io.ente.ensu.bindings.DictationSnapshot
+import io.ente.ensu.bindings.DictationStopReason
+import io.ente.ensu.bindings.DictationStream
+import io.ente.ensu.bindings.DictationStreamCallback
import io.ente.ensu.bindings.transcriptionModelAsset
import io.ente.ensu.bindings.voiceActivityModelAsset
+import io.ente.ensu.device.AndroidDeviceCapabilityProvider
+import io.ente.ensu.device.supportsLiveVoice
import io.ente.ensu.llm.LlmProvider
-import io.ente.ensu.llm.ModelMemoryUnavailable
-import java.io.ByteArrayOutputStream
+import io.ente.ensu.logging.FileLogRepository
+import io.ente.ensu.logging.LogLevel
import java.io.IOException
-import java.util.UUID
-import kotlin.coroutines.CoroutineContext
-import kotlin.coroutines.EmptyCoroutineContext
-import kotlin.coroutines.coroutineContext
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.math.log10
+import kotlin.math.sqrt
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.async
import kotlinx.coroutines.cancel
-import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
-import kotlinx.coroutines.ensureActive
-import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
internal sealed interface VoiceInputState {
data object Idle : VoiceInputState
+ data object Preparing : VoiceInputState
+
data object Recording : VoiceInputState
data class Downloading(val percent: Int?) : VoiceInputState
@@ -58,440 +62,464 @@ internal sealed interface VoiceInputState {
internal val VoiceInputState.isRecording: Boolean
get() = this is VoiceInputState.Recording
-internal val VoiceInputState.isWorking: Boolean
- get() =
- this is VoiceInputState.Recording ||
- this is VoiceInputState.Downloading ||
- this is VoiceInputState.Transcribing
-
-internal val VoiceInputState.blocksSend: Boolean
- get() = this is VoiceInputState.Recording || this is VoiceInputState.Transcribing
-
-internal fun VoiceInputState.statusText(): String? =
- when (this) {
- VoiceInputState.Idle -> null
- VoiceInputState.Recording -> "Listening..."
- is VoiceInputState.Downloading -> {
- val suffix = percent?.let { " ($it%)" }.orEmpty()
- "Downloading voice model...$suffix"
- }
- VoiceInputState.Transcribing -> null
- is VoiceInputState.Error -> message
- }
-
-@Composable
-internal fun rememberVoiceTranscriptionController(
- assetStore: AssetStore,
- llmProvider: LlmProvider,
- onTranscript: (String) -> Unit,
- onVoiceInputJob: (Job) -> Unit,
-): VoiceTranscriptionController {
- val lifecycleOwner = LocalLifecycleOwner.current
- val controller =
- remember(llmProvider) {
- VoiceTranscriptionController(
- assetStore,
- llmProvider,
- onTranscript,
- onVoiceInputJob,
- )
- }
- DisposableEffect(controller, lifecycleOwner) {
- val observer = LifecycleEventObserver { _, event ->
- if (event == Lifecycle.Event.ON_PAUSE) {
- controller.cancelActiveVoiceInput()
- }
- }
- lifecycleOwner.lifecycle.addObserver(observer)
- onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
- }
- DisposableEffect(controller) { onDispose { controller.dispose() } }
- return controller
-}
-
internal class VoiceTranscriptionController(
+ context: Context,
private val assetStore: AssetStore,
private val llmProvider: LlmProvider,
- private val onTranscript: (String) -> Unit,
+ private val logRepository: FileLogRepository,
private val onVoiceInputJob: (Job) -> Unit,
+ private val onInsert: (DictationInsertion, String) -> Boolean,
) {
+ private val diagnosticsEnabled =
+ context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0
+
+ private fun diagnostic(event: String) {
+ if (diagnosticsEnabled) {
+ logRepository.log(LogLevel.Info, event, tag = "EnsuVoice")
+ }
+ }
+
+ private val audioManager = context.getSystemService(AudioManager::class.java)
+ private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
+ private val usesLiveDictation =
+ supportsLiveVoice(
+ AndroidDeviceCapabilityProvider(context).chatCapability().totalMemoryBytes
+ )
private val modelAssets =
listOf(
transcriptionModelAsset(),
voiceActivityModelAsset(),
)
+ private var session: DictationSession? = null
+ private var stream: DictationStream? = null
+ private var capture: LiveMicrophone? = null
+ private var work: Job? = null
+ private var finishingWatchdog: Job? = null
+ private var attemptFinished = false
+ private var insertion: DictationInsertion? = null
+ private var foreground = true
+ private var generation = 0L
+ val sessionVersion: Long
+ get() = generation
+
+ private var initialRecordPending = false
+ private var awaitingPermission = false
+ private var startOnForeground = false
+
+ var active by mutableStateOf(false)
+ private set
- private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
- private val bufferLock = Any()
+ var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)
+ private set
- private var audioRecord: AudioRecord? = null
- private var preparingRecordingJob: Job? = null
- private var recordingJob: Job? = null
- private var transcriptionPreloadJob: Job? = null
- private var voiceOwner = UUID.randomUUID().toString()
- private var transientErrorJob: Job? = null
- private var recordedPcm = ByteArrayOutputStream()
- private var recordingSampleRate = preferredSampleRate
+ var snapshot: DictationSnapshot? by mutableStateOf(null)
+ private set
- var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)
+ private var notice: String? = null
+
+ var audioLevels: List<Float> by mutableStateOf(List(24) { 0f })
private set
- private fun launchVoiceWork(
- context: CoroutineContext = EmptyCoroutineContext,
- block: suspend CoroutineScope.() -> Unit,
- ): Job =
- scope.launch(context, start = CoroutineStart.LAZY, block = block).also {
- onVoiceInputJob(it)
- it.start()
- }
+ var composerCaret: Int? by mutableStateOf(null)
+
+ var hasActiveWork by mutableStateOf(false)
+ private set
- fun onPermissionDenied() {
- state = VoiceInputState.Error("Microphone permission is required for voice input.")
+ val canOpen: Boolean
+ get() = !active && !hasActiveWork
+
+ fun open(insertion: DictationInsertion) {
+ if (!canOpen) return
+ generation++
+ initialRecordPending = true
+ attemptFinished = false
+ this.insertion = insertion
+ session = DictationSession(live = usesLiveDictation)
+ snapshot = session?.snapshot()
+ notice = null
+ audioLevels = List(24) { 0f }
+ active = true
+ state = VoiceInputState.Idle
}
- @SuppressLint("MissingPermission")
- fun startRecording(canStartRecording: () -> Boolean = { true }) {
- transientErrorJob?.cancel()
- if (state.isWorking || preparingRecordingJob?.isActive == true) {
- return
- }
+ fun composerPreview(original: String): io.ente.ensu.bindings.DictationInsertionResult? {
+ val value = snapshot ?: return null
+ if (!active) return null
+ return runCatching { insertion?.preview(original, value.text + value.hypothesis) }
+ .getOrNull()
+ }
- preparingRecordingJob = launchVoiceWork {
- try {
- ensureTranscriptionModelDownloaded()
- if (!isActive || !canStartRecording()) {
- if (state is VoiceInputState.Downloading) {
- state = VoiceInputState.Idle
- }
- return@launchVoiceWork
- }
- beginRecording()
- preloadTranscriptionModel()
- } catch (error: CancellationException) {
- throw error
- } catch (error: UnsatisfiedLinkError) {
- Log.w(TAG, "Voice transcription is not available on this device", error)
- state =
- VoiceInputState.Error("Voice transcription is not available on this device.")
- } catch (error: TranscriptionException) {
- Log.w(TAG, "Voice model preparation failed: ${error.message}", error)
- state = VoiceInputState.Error(transcriptionErrorMessage(error))
- } catch (_: AssetDownloadException.Cancelled) {
- state = VoiceInputState.Idle
- } catch (error: AssetDownloadException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (_: LlmException.Cancelled) {
- state = VoiceInputState.Idle
- } catch (error: LlmException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (error: Throwable) {
- Log.w(
- TAG,
- "Voice model preparation failed: ${error.javaClass.simpleName}: ${error.message}",
- error,
- )
- state = VoiceInputState.Error("Could not transcribe voice input.")
- }
- }
+ fun takeInitialRecord(): Boolean = initialRecordPending.also { initialRecordPending = false }
+
+ fun permissionRequested() {
+ awaitingPermission = true
}
- @SuppressLint("MissingPermission")
- private fun beginRecording() {
- if (state is VoiceInputState.Recording || state is VoiceInputState.Transcribing) {
- return
+ fun permissionResult(granted: Boolean) {
+ awaitingPermission = false
+ if (granted) {
+ if (foreground) record() else startOnForeground = true
+ } else {
+ discard()
+ state =
+ VoiceInputState.Error("Microphone permission is required. Enable it in Settings.")
}
+ }
- synchronized(bufferLock) { recordedPcm = ByteArrayOutputStream() }
- recordingSampleRate = preferredSampleRate
- state = VoiceInputState.Recording
+ fun setForeground(active: Boolean) {
+ foreground = active
+ if (active) {
+ if (startOnForeground) {
+ startOnForeground = false
+ record()
+ } else session?.let { refresh(it, generation) }
+ } else if (!awaitingPermission && !startOnForeground) interrupt()
+ }
- recordingJob =
- launchVoiceWork(Dispatchers.IO) {
- var recorder: AudioRecord? = null
- try {
- val activeRecorder = createRecorder(recordingSampleRate)
- recorder = activeRecorder
- audioRecord = activeRecorder
- recordingSampleRate =
- activeRecorder.sampleRate.takeIf { it > 0 } ?: preferredSampleRate
- activeRecorder.startRecording()
- val buffer = ByteArray(readBufferSize(recordingSampleRate))
- while (coroutineContext.isActive) {
- val read = activeRecorder.read(buffer, 0, buffer.size)
- when {
- read > 0 ->
- synchronized(bufferLock) { recordedPcm.write(buffer, 0, read) }
- read < 0 -> throw IOException("Audio recorder failed with code $read")
+ fun record() {
+ val owner = session ?: return
+ if (!active || !foreground || hasActiveWork || attemptFinished) return
+ notice = null
+ state = VoiceInputState.Preparing
+ val token = generation
+ hasActiveWork = true
+ work =
+ scope
+ .launch {
+ var ownedCapture: LiveMicrophone? = null
+ var ownedStream: DictationStream? = null
+ try {
+ if (
+ !withContext(Dispatchers.IO) {
+ modelAssets.all { assetStore.isDownloaded(it) }
+ }
+ ) {
+ state = VoiceInputState.Downloading(null)
+ assetStore.download(modelAssets) { progress ->
+ scope.launch {
+ if (token == generation && state is VoiceInputState.Downloading)
+ state =
+ VoiceInputState.Downloading(progress.percentage.toInt())
+ }
+ }
}
- }
- } catch (error: Throwable) {
- if (error is CancellationException) {
+ if (token != generation || !foreground) return@launch
+ state = VoiceInputState.Preparing
+ val activeStream = DictationStream(owner, 16000u, 1u)
+ stream = activeStream
+ ownedStream = activeStream
+ val microphone =
+ LiveMicrophone(
+ activeStream,
+ audioManager,
+ onLevel = { level ->
+ scope.launch {
+ if (token == generation && state.isRecording)
+ audioLevels = audioLevels.drop(1) + level
+ }
+ },
+ ) {
+ scope.launch {
+ if (token == generation)
+ interrupt(
+ "Microphone interrupted. Review the text before continuing."
+ )
+ }
+ }
+ capture = microphone
+ ownedCapture = microphone
+ coroutineScope {
+ val hardwareReady = async(Dispatchers.IO) { microphone.prepare() }
+ val callback =
+ object : DictationStreamCallback {
+ override fun onReady() {
+ scope.launch {
+ try {
+ hardwareReady.await()
+ val value = owner.snapshot()
+ if (
+ token != generation ||
+ !foreground ||
+ !active ||
+ stream !== activeStream ||
+ !value.workerActive ||
+ value.phase != DictationPhase.LISTENING
+ )
+ return@launch
+ microphone.start(scope)
+ state = VoiceInputState.Recording
+ audioLevels = List(24) { 0f }
+ notice = null
+ refresh(owner, token)
+ } catch (_: Exception) {
+ if (token == generation)
+ interrupt(
+ "Could not start the microphone. Tap the microphone to start again."
+ )
+ }
+ }
+ }
+
+ override fun onChange() {
+ scope.launch { refresh(owner, token) }
+ }
+
+ override fun onCaptureClosed() {
+ microphone.stop()
+ scope.launch {
+ watchFinalization(owner, activeStream, token)
+ }
+ }
+ }
+ llmProvider.runLiveVoice(activeStream, callback)
+ }
+ } catch (error: CancellationException) {
throw error
- }
- Log.w(TAG, "Could not record microphone audio", error)
- withContext(Dispatchers.Main.immediate) {
- releaseVoice()
- if (state is VoiceInputState.Recording) {
- state = VoiceInputState.Error("Could not record microphone audio.")
+ } catch (error: Exception) {
+ ownedStream?.let { runCatching { it.abandon(DictationStopReason.FAILURE) } }
+ diagnostic(
+ if (error is DictationException.InvalidOperation)
+ "failure ${error.detail}"
+ else "failure ${error.javaClass.simpleName}"
+ )
+ if (token == generation && notice == null) {
+ notice = "Voice input stopped. Review the available text or try again."
}
- }
- } finally {
- runCatching {
- if (recorder?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
- recorder.stop()
+ } finally {
+ withContext(NonCancellable) { ownedCapture?.stopAndDrain() }
+ hasActiveWork = false
+ if (token == generation) {
+ finishingWatchdog?.cancel()
+ finishingWatchdog = null
+ capture = null
+ stream = null
+ attemptFinished = true
+ refresh(owner, token)
}
}
- recorder?.release()
- if (audioRecord === recorder) {
- audioRecord = null
- }
}
- }
+ .also(onVoiceInputJob)
}
- fun stopAndTranscribe() {
- if (state !is VoiceInputState.Recording) {
+ private fun refresh(owner: DictationSession, token: Long) {
+ if (token != generation || !active) return
+ val value = owner.snapshot()
+ snapshot = value
+ if (!attemptFinished) {
+ if (value.phase == DictationPhase.FINISHING) state = VoiceInputState.Transcribing
return
}
-
- state = VoiceInputState.Transcribing
- preparingRecordingJob?.cancel()
- preparingRecordingJob = null
- launchVoiceWork {
- recordingJob?.cancelAndJoin()
- recordingJob = null
-
- val pcm = synchronized(bufferLock) { recordedPcm.toByteArray() }
- if (pcm.size < minimumRecordingBytes(recordingSampleRate)) {
- releaseVoice()
- showTransientError("No speech captured.")
- return@launchVoiceWork
+ if (!foreground) return
+ val text = value.text + value.hypothesis
+ val added = text.isNotBlank() && insertion?.let { onInsert(it, text) } == true
+ val completedState =
+ when {
+ added && !value.incomplete -> VoiceInputState.Idle
+ text.isBlank() -> VoiceInputState.Error(notice ?: "No speech detected.")
+ else -> VoiceInputState.Error("Voice input stopped.")
}
-
- transcribeRecording(pcm, recordingSampleRate)
- }
+ active = false
+ snapshot = null
+ session = null
+ insertion = null
+ audioLevels = List(24) { 0f }
+ state = completedState
}
- fun cancelActiveVoiceInput() {
- releaseVoice()
- val wasPreparingRecording = preparingRecordingJob?.isActive == true
- preparingRecordingJob?.cancel()
- preparingRecordingJob = null
-
- if (state !is VoiceInputState.Recording) {
- if (wasPreparingRecording && state is VoiceInputState.Downloading) {
- state = VoiceInputState.Idle
- }
+ private fun watchFinalization(
+ owner: DictationSession,
+ finishingStream: DictationStream,
+ token: Long,
+ ) {
+ if (
+ token != generation ||
+ !active ||
+ stream !== finishingStream ||
+ !owner.snapshot().workerActive ||
+ finishingWatchdog != null
+ )
return
- }
-
- state = VoiceInputState.Idle
- recordingJob?.cancel()
- recordingJob = null
- stopActiveRecorder()
- synchronized(bufferLock) { recordedPcm.reset() }
- }
-
- fun dispose() {
- releaseVoice()
- transientErrorJob?.cancel()
- preparingRecordingJob?.cancel()
- scope.cancel()
- runCatching { audioRecord?.release() }
- audioRecord = null
- }
-
- private suspend fun transcribeRecording(pcm: ByteArray, sampleRate: Int) {
- val owner = voiceOwner
- try {
- ensureTranscriptionModelDownloaded()
- transcriptionPreloadJob?.join()
- coroutineContext.ensureActive()
- state = VoiceInputState.Transcribing
- val transcript = llmProvider.transcribe(sampleRate.toUInt(), pcm).trim()
-
- if (transcript.isBlank()) {
- showTransientError("No speech detected.")
- } else {
- onTranscript(transcript)
- state = VoiceInputState.Idle
- }
- } catch (error: CancellationException) {
- throw error
- } catch (error: UnsatisfiedLinkError) {
- Log.w(TAG, "Voice transcription is not available on this device", error)
- state = VoiceInputState.Error("Voice transcription is not available on this device.")
- } catch (error: TranscriptionException) {
- Log.w(TAG, "Voice transcription failed: ${error.message}", error)
- state = VoiceInputState.Error(transcriptionErrorMessage(error))
- } catch (error: ModelMemoryUnavailable) {
- state = VoiceInputState.Error(checkNotNull(error.message))
- } catch (_: AssetDownloadException.Cancelled) {
- state = VoiceInputState.Idle
- } catch (error: AssetDownloadException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (_: LlmException.Cancelled) {
- state = VoiceInputState.Idle
- } catch (error: LlmException) {
- Log.w(TAG, "Voice model download failed: ${error.message}", error)
- state = VoiceInputState.Error(downloadErrorMessage)
- } catch (error: Throwable) {
- Log.w(
- TAG,
- "Voice transcription failed: ${error.javaClass.simpleName}: ${error.message}",
- error,
+ finishingWatchdog = scope.launch {
+ delay(10_000)
+ if (
+ token != generation ||
+ !active ||
+ stream !== finishingStream ||
+ !owner.snapshot().workerActive
)
- state = VoiceInputState.Error("Could not transcribe voice input.")
- } finally {
- withContext(NonCancellable) { llmProvider.releaseVoice(owner) }
+ return@launch
+ capture?.stop()
+ runCatching { finishingStream.abandon(DictationStopReason.FAILURE) }
+ work?.cancel()
+ snapshot = owner.snapshot()
+ notice =
+ "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."
}
}
- private fun preloadTranscriptionModel() {
- voiceOwner = UUID.randomUUID().toString()
- val owner = voiceOwner
- transcriptionPreloadJob = launchVoiceWork {
- try {
- llmProvider.prewarmVoice(owner)
- } catch (error: CancellationException) {
- throw error
- } catch (error: Throwable) {
- Log.d(TAG, "Voice preload skipped", error)
- }
+ fun stop() {
+ val reason = DictationStopReason.DONE
+ runCatching { session?.stop(reason) }
+ snapshot = session?.snapshot()
+ if (snapshot?.workerActive != true) return
+ state = VoiceInputState.Transcribing
+ audioLevels = List(24) { 0f }
+ val active = stream
+ val microphone = capture
+ if (active != null) session?.let { watchFinalization(it, active, generation) }
+ scope.launch {
+ microphone?.stopAndDrain()
+ runCatching { active?.stop(reason) }
}
}
- private fun releaseVoice() {
- transcriptionPreloadJob?.cancel()
- transcriptionPreloadJob = null
- val owner = voiceOwner
- voiceOwner = UUID.randomUUID().toString()
- launchVoiceWork(NonCancellable) { llmProvider.releaseVoice(owner) }
- }
-
- private suspend fun ensureTranscriptionModelDownloaded() {
- val activeJob = coroutineContext[Job]
-
- if (withContext(Dispatchers.IO) { modelAssets.all { assetStore.isDownloaded(it) } }) {
+ fun interrupt(message: String = "Voice input stopped.") {
+ if (!active || attemptFinished) return
+ if (!hasActiveWork && stream == null) {
+ discard()
return
}
-
- setStateIfJobActive(activeJob, VoiceInputState.Downloading(null))
- assetStore.download(modelAssets) { progress ->
- postStateIfJobActive(
- activeJob,
- VoiceInputState.Downloading(progress.percentage.toInt().coerceIn(0, 100)),
- )
- }
+ capture?.stop()
+ runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }
+ work?.cancel()
+ notice = message
+ state = VoiceInputState.Transcribing
}
- private suspend fun setStateIfJobActive(activeJob: Job?, nextState: VoiceInputState) {
- if (activeJob?.isActive == false) {
- return
- }
- withContext(Dispatchers.Main.immediate) {
- if (activeJob?.isActive != false) {
- state = nextState
- }
- }
+ fun discard() {
+ awaitingPermission = false
+ startOnForeground = false
+ generation++
+ finishingWatchdog?.cancel()
+ finishingWatchdog = null
+ capture?.stop()
+ runCatching { stream?.abandon(DictationStopReason.DISCARD) }
+ work?.cancel()
+ active = false
+ state = VoiceInputState.Idle
+ snapshot = null
+ session = null
+ insertion = null
+ stream = null
+ capture = null
}
- private fun postStateIfJobActive(activeJob: Job?, nextState: VoiceInputState) {
- if (activeJob?.isActive == false) {
- return
- }
- scope.launch(Dispatchers.Main.immediate) {
- if (activeJob?.isActive != false) {
- state = nextState
- }
- }
+ fun dispose() {
+ discard()
+ scope.cancel()
}
+}
- private fun stopActiveRecorder() {
- val recorder = audioRecord ?: return
- runCatching {
- if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
- recorder.stop()
- }
- }
- }
+private class LiveMicrophone(
+ private val stream: DictationStream,
+ private val audioManager: AudioManager,
+ private val onLevel: (Float) -> Unit,
+ private val onInterrupted: () -> Unit,
+) {
+ private val stopped = AtomicBoolean(false)
+ @Volatile private var recorder: AudioRecord? = null
+ private var readJob: Job? = null
@SuppressLint("MissingPermission")
- private fun createRecorder(sampleRate: Int): AudioRecord {
- val minBufferSize =
+ fun prepare() {
+ if (stopped.get() || recorder != null) return
+ val minimum =
AudioRecord.getMinBufferSize(
- sampleRate,
+ 16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
)
- if (minBufferSize <= 0) {
- throw IOException("Audio recorder buffer unavailable")
+ if (minimum <= 0) throw IOException("Microphone format unavailable")
+ val active =
+ AudioRecord.Builder()
+ .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
+ .setAudioFormat(
+ AudioFormat.Builder()
+ .setSampleRate(16000)
+ .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
+ .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+ .build()
+ )
+ .setBufferSizeInBytes(maxOf(minimum * 2, 6400))
+ .build()
+ if (active.state != AudioRecord.STATE_INITIALIZED) {
+ active.release()
+ throw IOException("Microphone unavailable")
}
+ recorder = active
+ }
- return AudioRecord.Builder()
- .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
- .setAudioFormat(
- AudioFormat.Builder()
- .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
- .setSampleRate(sampleRate)
- .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
- .build()
- )
- .setBufferSizeInBytes(minBufferSize * 2)
- .build()
- .also { recorder ->
- if (recorder.state != AudioRecord.STATE_INITIALIZED) {
- recorder.release()
- throw IOException("Audio recorder failed to initialize")
+ @SuppressLint("MissingPermission")
+ fun start(scope: CoroutineScope) {
+ if (stopped.get()) return
+ val active = recorder ?: return
+ try {
+ active.startRecording()
+ if (active.recordingState != AudioRecord.RECORDSTATE_RECORDING)
+ throw IOException("Microphone did not start")
+ } catch (error: Exception) {
+ active.release()
+ recorder = null
+ throw error
+ }
+ var routedDevice = active.routedDevice?.id
+ readJob =
+ scope.launch(Dispatchers.IO) {
+ val buffer = ByteBuffer.allocateDirect(3200).order(ByteOrder.LITTLE_ENDIAN)
+ try {
+ while (!stopped.get()) {
+ buffer.clear()
+ val read = active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)
+ if (stopped.get() && read <= 0) break
+ if (
+ read <= 0 ||
+ audioManager.isMicrophoneMute ||
+ (android.os.Build.VERSION.SDK_INT >= 29 &&
+ active.activeRecordingConfiguration?.isClientSilenced ==
+ true) ||
+ (routedDevice != null && active.routedDevice?.id != routedDevice)
+ ) {
+ throw IOException("Microphone interrupted")
+ }
+ routedDevice = active.routedDevice?.id
+ buffer.position(0)
+ buffer.limit(read)
+ var energy = 0.0
+ for (index in 0 until read / 2) {
+ val sample = buffer.getShort(index * 2).toDouble() / 32768.0
+ energy += sample * sample
+ }
+ val rms = sqrt(energy / (read / 2).coerceAtLeast(1))
+ onLevel(
+ ((20 * log10(rms.coerceAtLeast(0.00001)) + 55) / 45)
+ .toFloat()
+ .coerceIn(0f, 1f)
+ )
+ stream.feed(buffer)
+ }
+ } catch (_: Exception) {
+ if (!stopped.get()) {
+ runCatching { stream.abandon(DictationStopReason.FAILURE) }
+ onInterrupted()
+ }
+ } finally {
+ stop()
+ active.release()
+ recorder = null
}
}
}
- private fun readBufferSize(sampleRate: Int): Int {
- val minBufferSize =
- AudioRecord.getMinBufferSize(
- sampleRate,
- AudioFormat.CHANNEL_IN_MONO,
- AudioFormat.ENCODING_PCM_16BIT,
- )
- return maxOf(minBufferSize, 4096)
- }
-
- private fun minimumRecordingBytes(sampleRate: Int): Int = sampleRate / 4 * bytesPerSample
-
- private val downloadErrorMessage =
- "Voice model download failed. Check your connection and try again."
-
- private fun transcriptionErrorMessage(error: TranscriptionException): String =
- when (error) {
- is TranscriptionException.NotDownloaded,
- is TranscriptionException.VadNotDownloaded,
- is TranscriptionException.Transcribe -> "Voice model could not be loaded."
- is TranscriptionException.InvalidAudio -> "Could not detect speech in this recording."
- is TranscriptionException.StorageFull ->
- "Not enough storage space. Please free up space and try again."
- is TranscriptionException.Io -> "Could not transcribe voice input."
- }
-
- private fun showTransientError(message: String) {
- val errorState = VoiceInputState.Error(message)
- transientErrorJob?.cancel()
- state = errorState
- transientErrorJob = scope.launch {
- delay(transientErrorMillis)
- if (state == errorState) {
- state = VoiceInputState.Idle
- }
+ suspend fun stopAndDrain() {
+ stop()
+ readJob?.join()
+ if (readJob == null) {
+ recorder?.release()
+ recorder = null
}
}
- private companion object {
- const val TAG = "VoiceInput"
- const val preferredSampleRate = 16_000
- const val bytesPerSample = 2
- const val transientErrorMillis = 10_000L
+ fun stop() {
+ if (stopped.compareAndSet(false, true)) runCatching { recorder?.stop() }
}
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
new file mode 100644
index 0000000000..b85adeef5c
--- /dev/null
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
@@ -0,0 +1,212 @@
+package io.ente.ensu.chat
+
+import android.Manifest
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.net.Uri
+import android.provider.Settings
+import androidx.activity.compose.rememberLauncherForActivityResult
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.compose.animation.core.animateFloatAsState
+import androidx.compose.animation.core.tween
+import androidx.compose.foundation.Canvas
+import androidx.compose.foundation.background
+import androidx.compose.foundation.clickable
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.Spacer
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.size
+import androidx.compose.foundation.shape.CircleShape
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.rounded.ArrowDownward
+import androidx.compose.material3.CircularProgressIndicator
+import androidx.compose.material3.Icon
+import androidx.compose.material3.IconButton
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.saveable.rememberSaveable
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.geometry.CornerRadius
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.res.painterResource
+import androidx.compose.ui.semantics.LiveRegionMode
+import androidx.compose.ui.semantics.contentDescription
+import androidx.compose.ui.semantics.liveRegion
+import androidx.compose.ui.semantics.semantics
+import androidx.compose.ui.unit.dp
+import androidx.core.content.ContextCompat
+import io.ente.ensu.bindings.DictationPhase
+import io.ente.ensu.designsystem.EnsuColor
+import io.ente.ensu.designsystem.EnsuSpacing
+import io.ente.ensu.designsystem.EnsuTypography
+import io.ente.ensu.designsystem.HugeIcons
+
+@Composable
+internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
+ val state = controller.state
+ if (!controller.active && state !is VoiceInputState.Error) return
+ val permissionNeeded =
+ (state as? VoiceInputState.Error)?.message?.contains("permission") == true
+ val context = LocalContext.current
+ val label =
+ when (state) {
+ is VoiceInputState.Downloading -> "Downloading"
+ VoiceInputState.Preparing,
+ VoiceInputState.Idle -> "Preparing"
+ VoiceInputState.Transcribing -> "Transcribing"
+ VoiceInputState.Recording ->
+ if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"
+ else "Listening"
+ is VoiceInputState.Error ->
+ if (permissionNeeded) "Microphone access"
+ else if (state.message == "No speech detected.") "No speech" else "Stopped"
+ }
+ Row(
+ Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),
+ verticalAlignment = Alignment.CenterVertically,
+ horizontalArrangement = Arrangement.spacedBy(12.dp),
+ ) {
+ Text(
+ label,
+ style = EnsuTypography.small,
+ color = EnsuColor.textMuted(),
+ modifier =
+ Modifier.semantics { liveRegion = LiveRegionMode.Polite }
+ .then(
+ if (permissionNeeded)
+ Modifier.clickable(onClickLabel = "Enable microphone") {
+ context.startActivity(
+ Intent(
+ Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
+ Uri.parse("package:${context.packageName}"),
+ )
+ )
+ }
+ else Modifier
+ ),
+ )
+ if (state.isRecording)
+ VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))
+ else Spacer(Modifier.weight(1f))
+ }
+}
+
+@Composable
+internal fun VoiceInputButton(
+ controller: VoiceTranscriptionController,
+ enabled: Boolean,
+ onOpen: () -> Unit,
+) {
+ val context = LocalContext.current
+ var permissionRequest by rememberSaveable { mutableStateOf<Long?>(null) }
+ val permission =
+ rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {
+ if (controller.active && permissionRequest == controller.sessionVersion)
+ controller.permissionResult(it)
+ permissionRequest = null
+ }
+ fun record() {
+ if (
+ ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
+ PackageManager.PERMISSION_GRANTED
+ )
+ controller.record()
+ else {
+ permissionRequest = controller.sessionVersion
+ controller.permissionRequested()
+ permission.launch(Manifest.permission.RECORD_AUDIO)
+ }
+ }
+ LaunchedEffect(controller.active, controller.sessionVersion) {
+ if (controller.active && controller.takeInitialRecord()) record()
+ }
+ val state = controller.state
+ val downloading = state as? VoiceInputState.Downloading
+ val preparing = state == VoiceInputState.Preparing
+ val transcribing = state == VoiceInputState.Transcribing
+ val capturing = state.isRecording
+ val canPress = if (controller.active) capturing else enabled && controller.canOpen
+ val description =
+ when {
+ downloading != null -> "Downloading voice model"
+ preparing -> "Preparing voice input"
+ transcribing -> "Transcribing"
+ capturing -> "Stop dictation"
+ else -> "Dictate"
+ }
+ IconButton(
+ onClick = {
+ when {
+ capturing -> controller.stop()
+ else -> onOpen()
+ }
+ },
+ enabled = canPress,
+ modifier = Modifier.size(44.dp).semantics { contentDescription = description },
+ ) {
+ when {
+ downloading != null || preparing || transcribing ->
+ Box(Modifier.size(28.dp), contentAlignment = Alignment.Center) {
+ if (downloading?.percent != null)
+ CircularProgressIndicator(
+ progress = { downloading.percent.coerceIn(0, 100) / 100f },
+ modifier = Modifier.size(28.dp),
+ strokeWidth = 2.dp,
+ color = EnsuColor.textPrimary(),
+ )
+ else
+ CircularProgressIndicator(
+ Modifier.size(28.dp),
+ strokeWidth = 2.dp,
+ color = EnsuColor.textPrimary(),
+ )
+ if (downloading != null)
+ Icon(Icons.Rounded.ArrowDownward, null, Modifier.size(14.dp))
+ }
+ capturing ->
+ Box(
+ Modifier.size(32.dp).background(EnsuColor.textPrimary(), CircleShape),
+ contentAlignment = Alignment.Center,
+ ) {
+ Icon(
+ painterResource(HugeIcons.StopIcon),
+ null,
+ Modifier.size(14.dp),
+ tint = EnsuColor.backgroundBase(),
+ )
+ }
+ else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(20.dp))
+ }
+ }
+}
+
+@Composable
+private fun VoiceWaveform(levels: List<Float>, modifier: Modifier) {
+ val animated = levels.map {
+ animateFloatAsState(it, tween(100), label = "Microphone level").value
+ }
+ val color = EnsuColor.textPrimary()
+ Canvas(modifier.semantics { contentDescription = "Microphone activity" }) {
+ val step = size.width / levels.size
+ val width = minOf(3.dp.toPx(), step * 0.6f)
+ animated.forEachIndexed { index, level ->
+ val height = 3.dp.toPx() + (size.height - 3.dp.toPx()) * level
+ drawRoundRect(
+ color.copy(alpha = 0.35f + level * 0.65f),
+ Offset(step * index + (step - width) / 2, (size.height - height) / 2),
+ Size(width, height),
+ CornerRadius(width / 2),
+ )
+ }
+ }
+}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
index 6bbb678100..832c392269 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/device/ChatDeviceCapability.kt
@@ -2,6 +2,9 @@ package io.ente.ensu.device
const val CHAT_MIN_RAM_BYTES: Long = 1_600_000_000L
+internal fun supportsLiveVoice(totalMemoryBytes: Long?): Boolean =
+ totalMemoryBytes != null && totalMemoryBytes > 4L * 1024 * 1024 * 1024
+
internal fun chatDeviceCapability(totalMemoryBytes: Long): ChatDeviceCapability =
when {
totalMemoryBytes <= 0 -> ChatDeviceCapability.Unknown
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index 3da32764c8..d97eabbea1 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -4,6 +4,9 @@ import android.os.SystemClock
import android.util.Log
import io.ente.ensu.assets.AssetStore
import io.ente.ensu.bindings.Asset
+import io.ente.ensu.bindings.DictationStopReason
+import io.ente.ensu.bindings.DictationStream
+import io.ente.ensu.bindings.DictationStreamCallback
import io.ente.ensu.bindings.KnowledgeEmbeddingConfig
import io.ente.ensu.bindings.LlmChatMessage as NativeChatMessage
import io.ente.ensu.bindings.LlmChatModelMemory
@@ -27,14 +30,18 @@ import io.ente.ensu.bindings.llmMemoryBudget
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
import io.ente.ensu.device.requireChatSupported
import io.ente.ensu.settings.IS_ENSU_PACKS_ENABLED
-import io.ente.ensu.toDirectByteBuffer
import java.io.File
+import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import kotlin.math.max
+import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
@@ -73,7 +80,6 @@ class LlmProvider(
private val activeDownloads = AtomicInteger()
private val embeddingAsset = knowledgeEmbeddingModelAsset()
private var chatWarmupOwner: String? = null
- private var voiceOwner: String? = null
@Volatile private var chatForeground = false
@Volatile private var appForeground = false
private val memoryPressure = ModelMemoryPressure(SystemClock::elapsedRealtime)
@@ -153,45 +159,45 @@ class LlmProvider(
}
}
- suspend fun prewarmVoice(owner: String) = withModelContext {
- withResources {
- currentCoroutineContext().ensureActive()
- if (!chatForeground || memoryPressure.suppressesOptionalWork()) return@withResources
- prepareVoiceResources()
- var completed = false
- try {
- transcriber.loadModel()
- currentCoroutineContext().ensureActive()
- voiceOwner = owner
- completed = true
- } finally {
- if (!completed) unloadTranscriptionModelIfLoaded()
+ suspend fun runLiveVoice(
+ stream: DictationStream,
+ callback: DictationStreamCallback,
+ ) = coroutineScope {
+ val exited = AtomicBoolean(false)
+ val cancellation =
+ launch(Dispatchers.Default, start = CoroutineStart.UNDISPATCHED) {
+ try {
+ awaitCancellation()
+ } finally {
+ if (!exited.get()) {
+ runCatching { stream.abandon(DictationStopReason.INTERRUPTION) }
+ }
+ }
}
- }
- }
-
- suspend fun releaseVoice(owner: String) =
- withContext(ioDispatcher) {
- modelLoadMutex.withLock { if (voiceOwner == owner) unloadTranscriptionModelIfLoaded() }
- }
-
- suspend fun transcribe(inputSampleRate: UInt, pcm: ByteArray): String = withModelContext {
- withResources {
- currentCoroutineContext().ensureActive()
- prepareVoiceResources()
- try {
- val result = transcriber.transcribe(inputSampleRate, pcm.toDirectByteBuffer())
- currentCoroutineContext().ensureActive()
- result
- } finally {
- unloadTranscriptionModelIfLoaded()
+ try {
+ withModelContext {
+ withResources {
+ currentCoroutineContext().ensureActive()
+ prepareVoiceResources()
+ try {
+ stream.run(transcriber, callback)
+ } finally {
+ unloadTranscriptionModelIfLoaded()
+ }
+ }
}
+ } catch (error: Throwable) {
+ runCatching { stream.abandon(DictationStopReason.FAILURE) }
+ throw error
+ } finally {
+ exited.set(true)
+ cancellation.cancel()
}
}
private fun prepareVoiceResources() {
releaseProjector()
- if (voiceOwner == null && !canAllocate(LlmMemoryOperation.VOICE)) {
+ if (!canAllocate(LlmMemoryOperation.VOICE)) {
unloadModel()
if (!hasRequiredWorkReserve()) throw ModelMemoryUnavailable()
}
@@ -238,12 +244,7 @@ class LlmProvider(
val result =
withResources(retainAfterCooldown = true) {
currentCoroutineContext().ensureActive()
- if (
- !chatForeground ||
- !appForeground ||
- voiceOwner != null ||
- !isChatModelReady(selection)
- )
+ if (!chatForeground || !appForeground || !isChatModelReady(selection))
return@withResources ChatWarmupResult.Skipped
val cooldown = memoryPressure.remainingCooldownMillis()
if (cooldown > 0) return@withResources ChatWarmupResult.RetryAfter(cooldown)
@@ -390,7 +391,6 @@ class LlmProvider(
if (
!chatForeground ||
memoryPressure.suppressesOptionalWork() ||
- voiceOwner != null ||
loadedContextLength(selection) == null
) {
throw ModelMemoryDeferred()
@@ -506,7 +506,7 @@ class LlmProvider(
checkCancellation()
deviceCapabilityProvider.chatCapability().requireChatSupported()
if (!isEmbeddingModelReady()) throw EmbeddingAssetInvalid()
- if (maintenance && (voiceOwner != null || memoryPressure.suppressesOptionalWork()))
+ if (maintenance && memoryPressure.suppressesOptionalWork())
throw ModelMemoryDeferred()
unloadTranscriptionModelIfLoaded()
releaseProjector()
@@ -608,7 +608,6 @@ class LlmProvider(
}
private fun unloadTranscriptionModelIfLoaded() {
- voiceOwner = null
runCatching {
transcriber.unloadModel()
}
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
index d1480cefd7..5ba9a19595 100644
--- a/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/device/ChatDeviceCapabilityTest.kt
@@ -7,6 +7,15 @@ import org.junit.Assert.assertTrue
import org.junit.Test
class ChatDeviceCapabilityTest {
+ @Test
+ fun liveVoiceRequiresMoreThanFourGiB() {
+ val limit = 4L * 1024 * 1024 * 1024
+ for (bytes in listOf(null, -1L, 0L, 4_000_000_000L, limit)) {
+ assertFalse(supportsLiveVoice(bytes))
+ }
+ for (bytes in listOf(limit + 1, 8_000_000_000L)) assertTrue(supportsLiveVoice(bytes))
+ }
+
@Test
fun minimumRamBlocksOnlyKnownReadingsBelowTheFloor() {
val belowMinimum = chatDeviceCapability(1_599_999_999L)
diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swift
index a916c7b661..c4ada28a33 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift
@@ -161,9 +161,10 @@ struct ChatView: View {
viewModel.setAppForeground(newValue != .background)
viewModel.notesStore.setForeground(newValue == .active)
if newValue == .active {
+ viewModel.voiceTranscriber.setForeground(true)
viewModel.refreshModelDownloadInfo()
} else {
- viewModel.cancelVoiceInput()
+ viewModel.voiceTranscriber.setForeground(false)
}
}
}
@@ -204,6 +205,7 @@ struct ChatView: View {
}
.onChange(of: viewState.showSettings) { isPresented in
if isPresented {
+ viewModel.voiceTranscriber.interrupt()
viewState.didDismissKeyboard = true
isInputFocused = false
} else if viewState.pendingSignInRequest {
@@ -362,7 +364,7 @@ struct ChatView: View {
}
.padding(.bottom, keyboard.isVisible ? keyboard.height : 0)
.zIndex(1)
- } else if viewModel.isModelDownloaded {
+ } else {
let shouldAutoFocus = shouldAutoFocusInput
MessageInputView(
@@ -374,7 +376,7 @@ struct ChatView: View {
isDownloading: viewModel.isDownloading,
editingMessage: editingMessage,
isProcessingAttachments: viewModel.isProcessingAttachments,
- voiceInputState: viewModel.voiceInputState,
+ voiceService: viewModel.voiceTranscriber,
moveCursorToEndToken: viewModel.draftCursorMoveToken,
onSend: {
viewState.didDismissKeyboard = true
@@ -392,6 +394,11 @@ struct ChatView: View {
isInputFocused = false
viewModel.toggleVoiceInput()
},
+ voiceInsertionCaret: viewModel.voiceInsertionCaret,
+ onComposerSelection: { range, composing in
+ viewModel.composerSelection = range
+ viewModel.composerIsComposing = composing
+ },
onAddImage: { data, name in
viewModel.addImageAttachment(data: data, fileName: name)
},
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index a992d83679..d7711f92f7 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -71,7 +71,14 @@ final class ChatViewModel: ObservableObject {
@Published private(set) var isSendPending: Bool = false
@Published var isDownloading: Bool = false
@Published var isProcessingAttachments: Bool = false
- @Published var draftText: String = ""
+ @Published var draftText: String = "" {
+ didSet {
+ if draftText != oldValue {
+ composerRevision += 1
+ voiceInsertionCaret = nil
+ }
+ }
+ }
@Published var draftAttachments: [ChatAttachment] = []
private var draftImageAttachmentCount: Int {
draftAttachments.filter { $0.kind == .image }.count
@@ -85,7 +92,6 @@ final class ChatViewModel: ObservableObject {
@Published var deviceCapability: ChatDeviceCapability = .unknown
@Published var showUnsupportedDeviceDialog: Bool = false
@Published var generationErrorMessage: String?
- @Published var voiceInputState: VoiceInputState = .idle
@Published var draftCursorMoveToken = UUID()
let knowledgeStore: KnowledgeStore
let notesStore: NotesStore
@@ -98,7 +104,12 @@ final class ChatViewModel: ObservableObject {
private let provider: LlmProvider
private let knowledgeEmbedding: KnowledgeEmbeddingConfig
private let knowledgeProvider: KnowledgeProvider
- private let voiceTranscriber: VoiceTranscriptionService
+ let voiceTranscriber: VoiceTranscriptionService
+ private var composerRevision: UInt64 = 0
+ private var emptyComposerId = UUID()
+ var composerSelection = NSRange(location: 0, length: 0)
+ var composerIsComposing = false
+ @Published var voiceInsertionCaret: Int?
private var chatDb: EnsuDb
private let attachmentsDir: URL
private let chatDbPath: String
@@ -116,7 +127,6 @@ final class ChatViewModel: ObservableObject {
private var pendingSendTask: Task<Void, Never>?
private var pendingSendId: UUID?
private var modelDownloadTask: Task<Void, Never>?
- private var voiceTransientErrorTask: Task<Void, Never>?
private var sharedModelReadyTask: Task<Void, Error>?
private var sharedModelReadyTaskId: UUID?
private var sharedModelReadyKey: ModelReadyKey?
@@ -142,9 +152,18 @@ final class ChatViewModel: ObservableObject {
// Must initialize after the asset store migrates the persisted selection.
self.modelSettings = ModelSettingsStore.shared
let transcription = transcriptionModelAsset()
- let transcriber = Transcriber(
+ var info = utsname()
+ uname(&info)
+ let machine = withUnsafeBytes(of: &info.machine) { bytes in
+ String(decoding: bytes.prefix { $0 != 0 }, as: UTF8.self)
+ }
+ let transcriber = Transcriber.withCache(
modelDir: assetStore.assetDir(transcription).path,
- vadModelPath: assetStore.voiceActivityModelPath().path
+ vadModelPath: assetStore.voiceActivityModelPath().path,
+ cacheDirectory: (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)
+ .first
+ ?? FileManager.default.temporaryDirectory).path,
+ deviceIdentity: "\(machine)|\(ProcessInfo.processInfo.operatingSystemVersionString)"
)
let provider = LlmProvider(
assetStore: assetStore, transcriber: transcriber,
@@ -220,14 +239,7 @@ final class ChatViewModel: ObservableObject {
loadMessagesFromDb(for: current)
}
- voiceTranscriber.onTaskActivityChanged = { [weak self] active in
- guard let self else { return }
- if active {
- self.cancelChatWarmup(releaseLoadedModel: false)
- } else {
- self.refreshChatWarmup()
- }
- }
+ voiceTranscriber.onActivityChanged = { [weak self] in self?.refreshChatWarmup() }
refreshDeviceCapability()
refreshModelDownloadInfo()
Task {
@@ -260,6 +272,7 @@ final class ChatViewModel: ObservableObject {
}
func handleMemoryPressure() {
+ voiceTranscriber.interrupt(message: "Voice input paused because memory is low.")
sessionSummaryTask?.cancel()
warmupSuppressed = true
cancelChatWarmup()
@@ -281,7 +294,7 @@ final class ChatViewModel: ObservableObject {
let key = modelReadyKey(for: selection)
let eligible =
appForeground && chatActive && !warmupSuppressed && !isChatUnsupported
- && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks
+ && !voiceTranscriber.active && !voiceTranscriber.hasActiveTasks
if let attempt = warmupAttempt, attempt.selection != key { cancelChatWarmup() }
guard eligible, warmupAttempt == nil, !isGenerating, !isDownloading,
provider.isChatModelReady(selection)
@@ -307,7 +320,7 @@ final class ChatViewModel: ObservableObject {
&& modelReadyKey(for: modelSettings.currentSelection()) == key
&& appForeground && chatActive && !warmupSuppressed
&& !isChatUnsupported && !isGenerating && !isDownloading
- && !voiceInputState.isWorking && !voiceTranscriber.hasActiveTasks
+ && !voiceTranscriber.active && !voiceTranscriber.hasActiveTasks
},
operation: {
try await provider.prewarmChatModelIfDownloaded(selection, owner: owner)
@@ -472,6 +485,7 @@ final class ChatViewModel: ObservableObject {
return
}
if currentSessionId == session.id {
+ cancelVoiceInput()
discardUnstoredAttachments(draftAttachments)
}
for id in attachmentIds {
@@ -501,6 +515,7 @@ final class ChatViewModel: ObservableObject {
return
}
guard message.role == .user else { return }
+ cancelVoiceInput()
resetGenerationState()
editingMessageId = message.id
draftText = message.text
@@ -516,76 +531,45 @@ final class ChatViewModel: ObservableObject {
}
func toggleVoiceInput() {
+ guard !isGenerating, !isDownloading, !isSendPending, editingMessageId == nil,
+ !composerIsComposing, !voiceTranscriber.active, !voiceTranscriber.hasActiveTasks
+ else { return }
+ let range = composerSelection
+ let text = draftText as NSString
+ guard range.location >= 0, range.length >= 0, range.location <= text.length,
+ range.length <= text.length - range.location,
+ [range.location, NSMaxRange(range)].allSatisfy({
+ $0 == text.length || text.rangeOfComposedCharacterSequence(at: $0).location == $0
+ })
+ else { return }
sessionSummaryTask?.cancel()
cancelChatWarmup(releaseLoadedModel: false)
- if voiceInputState.isRecording {
- voiceTranscriber.stopAndTranscribe(
- onState: { [weak self] state in
- self?.setVoiceInputState(state)
- },
- onTranscript: { [weak self] transcript in
- self?.appendVoiceTranscript(transcript)
- }
- )
- return
- }
-
- guard !isGenerating,
- !isDownloading,
- editingMessageId == nil
- else {
- return
+ let destination = (currentSessionId ?? emptyComposerId).uuidString
+ let revision = composerRevision
+ let insertion = DictationInsertion(
+ destination: destination, revision: revision,
+ startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))
+ voiceTranscriber.onInsert = { [weak self] insertion, text in
+ guard let self, self.appForeground else { return false }
+ do {
+ let result = try insertion.apply(
+ destination: (self.currentSessionId ?? self.emptyComposerId).uuidString,
+ revision: self.composerRevision, text: self.draftText, voiceText: text,
+ foreground: self.appForeground, keyboardComposing: self.composerIsComposing)
+ self.draftText = result.text
+ self.composerRevision = result.revision
+ self.voiceInsertionCaret = Int(result.caretUtf16)
+ self.draftCursorMoveToken = UUID()
+ return true
+ } catch { return false }
}
-
- let voiceSessionId = currentSessionId
- voiceTranscriber.startRecording(
- onState: { [weak self] state in
- self?.setVoiceInputState(state)
- },
- shouldStartRecording: { [weak self] in
- guard let self else { return false }
- return self.currentSessionId == voiceSessionId && !self.isGenerating
- && !self.isDownloading && self.editingMessageId == nil
- }
- )
+ voiceTranscriber.open(insertion: insertion)
}
func cancelVoiceInput() {
voiceTranscriber.cancel()
- setVoiceInputState(.idle)
- }
-
- private func setVoiceInputState(_ state: VoiceInputState) {
- defer { refreshChatWarmup() }
- voiceTransientErrorTask?.cancel()
- voiceTransientErrorTask = nil
- voiceInputState = state
-
- guard state.isNoSpeechError else { return }
- voiceTransientErrorTask = Task { @MainActor in
- do {
- try await Task.sleep(nanoseconds: 10_000_000_000)
- } catch {
- return
- }
- if self.voiceInputState == state {
- self.voiceInputState = .idle
- }
- self.voiceTransientErrorTask = nil
- }
- }
-
- private func appendVoiceTranscript(_ transcript: String) {
- let cleaned = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
- guard !cleaned.isEmpty else { return }
-
- let trimmedDraft = draftText.trimmingCharacters(in: .whitespacesAndNewlines)
- if trimmedDraft.isEmpty {
- draftText = cleaned
- } else {
- draftText = "\(trimmedDraft) \(cleaned)"
- }
- draftCursorMoveToken = UUID()
+ emptyComposerId = UUID()
+ voiceInsertionCaret = nil
}
func addImageAttachment(data: Data, fileName: String?) {
@@ -731,6 +715,7 @@ final class ChatViewModel: ObservableObject {
}
func sendDraft() {
+ guard !voiceTranscriber.active else { return }
guard !isGenerating && !isDownloading && !isSendPending else { return }
guard !isChatUnsupported else {
showUnsupportedDeviceDialog = true
@@ -1485,7 +1470,10 @@ final class ChatViewModel: ObservableObject {
let details =
"session=\(userNode.sessionId.uuidString) promptLen=\(prompt.text.count) responseLen=\(snapshot.text.count) tokens=\(snapshot.tokenCount)"
logger.error("Generation failed", error, details: details)
- if case let ConversationError.Other(detail) = error {
+ if case ConversationError.CurrentInputTooLarge = error {
+ generationErrorMessage =
+ "This message is too long for the model, even without earlier messages. Shorten it or split it into smaller messages. Your full text is still available to edit or copy."
+ } else if case let ConversationError.Other(detail) = error {
generationErrorMessage = detail
} else if case ConversationError.Stale = error {
generationErrorMessage = "Conversation changed. Retry the message."
@@ -1607,6 +1595,9 @@ final class ChatViewModel: ObservableObject {
sessionId: parent.sessionId, promptText: prompt.text,
promptImageCount: prompt.imageFiles.count, currentMessageId: parent.id,
limits: limits, systemPrompt: systemPrompt)
+ if normalHistory.currentInputTooLarge {
+ throw ConversationError.CurrentInputTooLarge
+ }
if normalHistory.wasTrimmed && overflowBypassMessageId != parent.id {
return .overflow(
inputTokens: normalHistory.inputTokens,
@@ -2512,6 +2503,7 @@ final class ChatViewModel: ObservableObject {
let inputTokens: Int
let inputBudget: Int
let wasTrimmed: Bool
+ let currentInputTooLarge: Bool
}
private struct GenerationLimits {
@@ -2580,7 +2572,8 @@ final class ChatViewModel: ObservableObject {
if remaining <= 0 || historyMessages.isEmpty {
return HistorySelection(
messages: [], inputTokens: inputTokens, inputBudget: inputBudget,
- wasTrimmed: inputTokens > inputBudget)
+ wasTrimmed: inputTokens > inputBudget,
+ currentInputTooLarge: systemTokens + promptTokens > inputBudget)
}
let quantum = max(1, inputBudget / 4)
@@ -2610,7 +2603,7 @@ final class ChatViewModel: ObservableObject {
return HistorySelection(
messages: selected, inputTokens: inputTokens, inputBudget: inputBudget,
- wasTrimmed: inputTokens > inputBudget)
+ wasTrimmed: inputTokens > inputBudget, currentInputTooLarge: false)
}
private func resolveMaxOutputTokens(contextLength: Int) -> Int {
diff --git a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
index 17f78d90bd..f9bf3bd0d8 100644
--- a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
+++ b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
@@ -12,12 +12,14 @@ struct MessageInputView: View {
let isDownloading: Bool
let editingMessage: RenderedChatMessage?
let isProcessingAttachments: Bool
- let voiceInputState: VoiceInputState
+ @ObservedObject var voiceService: VoiceTranscriptionService
let moveCursorToEndToken: UUID
let onSend: () -> Void
let onStop: () -> Void
let onCancelEdit: () -> Void
let onVoiceInput: () -> Void
+ let voiceInsertionCaret: Int?
+ let onComposerSelection: (NSRange, Bool) -> Void
let onAddImage: (Data, String?) -> Void
let onAddDocument: (URL) -> Void
let onRemoveAttachment: (ChatAttachment) -> Void
@@ -38,7 +40,8 @@ struct MessageInputView: View {
private var isSendEnabled: Bool {
let hasContent =
!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !attachments.isEmpty
- return hasContent && !isDownloading && !isSendPending && !voiceInputState.blocksSend
+ return hasContent && !isDownloading && !isSendPending
+ && !voiceService.active
}
private var isImageAttachmentLimitReached: Bool {
@@ -46,6 +49,9 @@ struct MessageInputView: View {
}
var body: some View {
+ let voicePreview = voiceService.composerPreview(original: text)
+ let composerText = voicePreview?.text ?? text
+
ZStack(alignment: .bottomTrailing) {
VStack(alignment: .leading, spacing: EnsuSpacing.sm) {
if let editingMessage {
@@ -92,37 +98,11 @@ struct MessageInputView: View {
.padding(.horizontal, EnsuSpacing.sm)
}
- if let voiceStatus = voiceInputState.statusText {
- HStack(spacing: EnsuSpacing.sm) {
- switch voiceInputState {
- case .downloading, .transcribing:
- ProgressView()
- .scaleEffect(0.8)
- case .recording:
- Image("Mic02Icon")
- .resizable()
- .scaledToFit()
- .frame(width: 14, height: 14)
- .foregroundStyle(EnsuColor.stopButton)
- case .error:
- Image(systemName: "exclamationmark.circle")
- .font(.system(size: 12, weight: .semibold))
- .foregroundStyle(EnsuColor.stopButton)
- case .idle:
- EmptyView()
- }
- Text(voiceStatus)
- .font(EnsuTypography.small)
- .foregroundStyle(
- voiceInputState.isError
- ? EnsuColor.stopButton : EnsuColor.textMuted)
- Spacer()
- }
- .padding(.horizontal, EnsuSpacing.sm)
- }
+ VoiceInputControls(service: voiceService)
+ .padding(.horizontal, EnsuSpacing.pageHorizontal)
HStack(alignment: .bottom, spacing: EnsuSpacing.xs) {
- if supportsImageAttachments {
+ if supportsImageAttachments && !voiceService.active {
ActionButton(
icon: "ImagePlusIcon", tooltip: "Add photos",
color: EnsuColor.textPrimary, size: 32
@@ -154,7 +134,10 @@ struct MessageInputView: View {
}
MessageTextView(
- text: $text,
+ text: Binding(
+ get: { composerText },
+ set: { if !voiceService.active { text = $0 } }
+ ),
isFocused: $isFocused,
moveCursorToEndToken: moveCursorToEndToken,
onSubmit: {
@@ -164,11 +147,16 @@ struct MessageInputView: View {
hideKeyboard()
onDismissKeyboard()
}
- }
+ },
+ insertionCaret: voicePreview.map { Int($0.caretUtf16) }
+ ?? voiceInsertionCaret,
+ onSelectionChange: onComposerSelection,
+ readOnly: voiceService.active,
+ maximumLines: voiceService.active ? 8 : 5
)
.padding(.vertical, 6)
.overlay(alignment: .topLeading) {
- if text.isEmpty {
+ if composerText.isEmpty {
Text(placeholder)
.font(EnsuTypography.message)
.foregroundStyle(Color(uiColor: .placeholderText))
@@ -184,49 +172,16 @@ struct MessageInputView: View {
}
if editingMessage == nil {
- let isVoiceBusy = voiceInputState.isTranscriptionBusy
- let canUseVoice =
- voiceInputState.isRecording
- || (!isGenerating && !isDownloading && !isVoiceBusy)
-
- Button {
- if voiceInputState.isRecording {
- hapticWarning()
- } else {
- hapticTap()
- isFocused = false
- hideKeyboard()
- onDismissKeyboard()
- }
+ VoiceInputButton(
+ service: voiceService,
+ enabled: !isGenerating && !isDownloading && !isSendPending
+ ) {
+ hapticTap()
+ isFocused = false
+ hideKeyboard()
+ onDismissKeyboard()
onVoiceInput()
- } label: {
- if isVoiceBusy {
- ProgressView()
- .scaleEffect(0.75)
- .frame(width: 32, height: 32)
- } else if voiceInputState.isRecording {
- Image("StopIcon")
- .resizable()
- .scaledToFit()
- .frame(
- width: composerIconSize, height: composerIconSize
- )
- .foregroundStyle(EnsuColor.sendButton)
- .frame(width: 32, height: 32)
- } else {
- Image("Mic02Icon")
- .resizable()
- .scaledToFit()
- .frame(width: composerIconSize, height: composerIconSize)
- .foregroundStyle(
- canUseVoice
- ? EnsuColor.textPrimary : EnsuColor.textMuted
- )
- .frame(width: 32, height: 32)
- }
}
- .disabled(!canUseVoice)
- .padding(.leading, EnsuSpacing.xs)
}
Button {
diff --git a/apple/apps/ensu/Ensu/Chat/MessageTextView.swift b/apple/apps/ensu/Ensu/Chat/MessageTextView.swift
index f1e9790a83..39529ca092 100644
--- a/apple/apps/ensu/Ensu/Chat/MessageTextView.swift
+++ b/apple/apps/ensu/Ensu/Chat/MessageTextView.swift
@@ -6,6 +6,10 @@ struct MessageTextView: UIViewRepresentable {
@Binding var isFocused: Bool
let moveCursorToEndToken: UUID
let onSubmit: () -> Void
+ var insertionCaret: Int? = nil
+ var onSelectionChange: ((NSRange, Bool) -> Void)? = nil
+ var readOnly = false
+ var maximumLines: CGFloat = 5
func makeCoordinator() -> Coordinator {
Coordinator(self)
@@ -32,13 +36,23 @@ struct MessageTextView: UIViewRepresentable {
func updateUIView(_ uiView: UITextView, context: Context) {
context.coordinator.parent = self
+ uiView.isEditable = !readOnly
if uiView.text != text {
uiView.undoManager?.removeAllActions()
uiView.text = text
+ if readOnly, !uiView.isDragging, !uiView.isDecelerating {
+ let caret = min(
+ insertionCaret ?? (text as NSString).length, (text as NSString).length)
+ uiView.selectedRange = NSRange(location: caret, length: 0)
+ uiView.scrollRangeToVisible(uiView.selectedRange)
+ }
}
if context.coordinator.lastCursorToken != moveCursorToEndToken {
context.coordinator.lastCursorToken = moveCursorToEndToken
- uiView.selectedRange = NSRange(location: (text as NSString).length, length: 0)
+ uiView.selectedRange = NSRange(
+ location: min(
+ insertionCaret ?? (text as NSString).length, (text as NSString).length),
+ length: 0)
}
if isFocused != uiView.isFirstResponder {
DispatchQueue.main.async { [weak uiView, weak coordinator = context.coordinator] in
@@ -55,12 +69,14 @@ struct MessageTextView: UIViewRepresentable {
func sizeThatFits(_ proposal: ProposedViewSize, uiView: UITextView, context: Context) -> CGSize?
{
guard let width = proposal.width, width > 0, width.isFinite else { return nil }
- return CGSize(width: width, height: Self.height(for: text, width: width))
+ return CGSize(
+ width: width,
+ height: Self.height(for: text, width: width, maximumLines: maximumLines))
}
- private static func height(for text: String, width: CGFloat) -> CGFloat {
+ private static func height(for text: String, width: CGFloat, maximumLines: CGFloat) -> CGFloat {
let font = UIFont.systemFont(ofSize: 15)
- let maximumHeight = ceil(font.lineHeight * 5)
+ let maximumHeight = ceil(font.lineHeight * maximumLines)
let scalars = text.unicodeScalars.prefix(1025)
guard scalars.count <= 1024 else { return maximumHeight }
let sample = String(String.UnicodeScalarView(scalars)) + "\u{200B}"
@@ -97,6 +113,11 @@ struct MessageTextView: UIViewRepresentable {
func textViewDidChange(_ textView: UITextView) {
parent.text = textView.text
+ parent.onSelectionChange?(textView.selectedRange, textView.markedTextRange != nil)
+ }
+
+ func textViewDidChangeSelection(_ textView: UITextView) {
+ parent.onSelectionChange?(textView.selectedRange, textView.markedTextRange != nil)
}
func textViewDidBeginEditing(_ textView: UITextView) {
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
new file mode 100644
index 0000000000..5ba2e8d49a
--- /dev/null
+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
@@ -0,0 +1,121 @@
+import SwiftUI
+
+struct VoiceInputControls: View {
+ @ObservedObject var service: VoiceTranscriptionService
+ @Environment(\.openURL) private var openURL
+ @Environment(\.accessibilityReduceMotion) private var reduceMotion
+
+ private var status: String {
+ switch service.state {
+ case .downloading: return "Downloading"
+ case .preparing: return "Preparing"
+ case .transcribing: return "Transcribing"
+ case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"
+ case .error(let message):
+ if message.contains("permission") { return "Microphone access" }
+ return message == "No speech detected." ? "No speech" : "Stopped"
+ case .idle: return "Preparing"
+ }
+ }
+
+ var body: some View {
+ if service.active || service.state.isError {
+ HStack(spacing: 12) {
+ Text(status).font(EnsuTypography.small).foregroundStyle(EnsuColor.textMuted)
+ .accessibilityHint(service.notice ?? "")
+ .onTapGesture {
+ if case .error(let message) = service.state,
+ message.contains("permission"),
+ let url = URL(string: UIApplication.openSettingsURLString)
+ {
+ openURL(url)
+ }
+ }
+ if service.state.isRecording {
+ waveform.frame(height: 24)
+ } else {
+ Spacer(minLength: 0)
+ }
+ }
+ }
+ }
+
+ private var waveform: some View {
+ GeometryReader { geometry in
+ HStack(alignment: .center, spacing: 0) {
+ ForEach(service.audioLevels.indices, id: \.self) { index in
+ let level = service.audioLevels[index]
+ Capsule()
+ .fill(EnsuColor.textPrimary.opacity(0.35 + Double(level) * 0.65))
+ .frame(
+ width: min(3, geometry.size.width / 48), height: 3 + CGFloat(level) * 21
+ )
+ .frame(
+ width: geometry.size.width / CGFloat(service.audioLevels.count),
+ height: 24)
+ }
+ }
+ .animation(reduceMotion ? nil : .easeOut(duration: 0.1), value: service.audioLevels)
+ }
+ .accessibilityElement(children: .ignore)
+ .accessibilityLabel("Microphone activity")
+ }
+}
+
+struct VoiceInputButton: View {
+ @ObservedObject var service: VoiceTranscriptionService
+ let enabled: Bool
+ let onOpen: () -> Void
+
+ private var canPress: Bool {
+ service.active ? service.state.isRecording : enabled && !service.hasActiveTasks
+ }
+
+ private var label: String {
+ switch service.state {
+ case .downloading: return "Downloading voice model"
+ case .preparing: return "Preparing voice input"
+ case .transcribing: return "Transcribing"
+ case .recording: return "Stop dictation"
+ case .idle, .error: return "Dictate"
+ }
+ }
+
+ var body: some View {
+ Button {
+ if service.state.isRecording { service.stop() } else { onOpen() }
+ } label: {
+ ZStack {
+ switch service.state {
+ case .downloading(let percent):
+ if let percent {
+ Circle().stroke(EnsuColor.textMuted.opacity(0.2), lineWidth: 2)
+ Circle().trim(from: 0, to: CGFloat(min(100, max(0, percent))) / 100)
+ .stroke(
+ EnsuColor.textPrimary,
+ style: StrokeStyle(lineWidth: 2, lineCap: .round)
+ )
+ .rotationEffect(.degrees(-90))
+ } else {
+ ProgressView().controlSize(.small)
+ }
+ Image(systemName: "arrow.down").font(.system(size: 11, weight: .semibold))
+ case .preparing, .transcribing:
+ ProgressView().controlSize(.small)
+ case .recording:
+ Circle().fill(EnsuColor.textPrimary).frame(width: 32, height: 32)
+ RoundedRectangle(cornerRadius: 2).fill(EnsuColor.backgroundBase)
+ .frame(width: 12, height: 12)
+ case .idle, .error:
+ Image("Mic02Icon").resizable().scaledToFit().frame(width: 20, height: 20)
+ }
+ }
+ .frame(width: 28, height: 28)
+ .frame(width: 44, height: 44)
+ .contentShape(Rectangle())
+ }
+ .foregroundStyle(canPress ? EnsuColor.textPrimary : EnsuColor.textMuted)
+ .disabled(!canPress)
+ .accessibilityLabel(label)
+ }
+}
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 3748339cd9..029d4c1a4d 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -1,45 +1,17 @@
import Foundation
import AVFoundation
+import Combine
enum VoiceInputState: Equatable {
case idle
+ case preparing
case recording
case downloading(percent: Int?)
case transcribing
case error(String)
var isRecording: Bool {
- if case .recording = self {
- return true
- }
- return false
- }
-
- var isWorking: Bool {
- switch self {
- case .recording, .downloading, .transcribing:
- return true
- case .idle, .error:
- return false
- }
- }
-
- var isTranscriptionBusy: Bool {
- switch self {
- case .downloading, .transcribing:
- return true
- case .idle, .recording, .error:
- return false
- }
- }
-
- var blocksSend: Bool {
- switch self {
- case .recording, .transcribing:
- return true
- case .idle, .downloading, .error:
- return false
- }
+ self == .recording
}
var isError: Bool {
@@ -49,394 +21,493 @@ enum VoiceInputState: Equatable {
return false
}
- var isNoSpeechError: Bool {
- switch self {
- case let .error(message):
- return message == "No speech detected." || message == "No speech captured."
- case .idle, .recording, .downloading, .transcribing:
- return false
- }
- }
-
- var statusText: String? {
- switch self {
- case .idle:
- return nil
- case .recording:
- return "Listening..."
- case let .downloading(percent):
- if let percent {
- return "Downloading voice model... (\(percent)%)"
- }
- return "Downloading voice model..."
- case .transcribing:
- return nil
- case let .error(message):
- return message
- }
- }
}
@MainActor
-final class VoiceTranscriptionService {
- typealias StateHandler = @MainActor @Sendable (VoiceInputState) -> Void
- typealias TranscriptHandler = @MainActor @Sendable (String) -> Void
-
- var onTaskActivityChanged: (@MainActor @Sendable (Bool) -> Void)?
+final class VoiceTranscriptionService: ObservableObject {
+ @Published private(set) var active = false {
+ didSet { onActivityChanged?() }
+ }
+ @Published private(set) var snapshot: DictationSnapshot?
+ @Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)
+ @Published private(set) var notice: String?
+ @Published private(set) var state: VoiceInputState = .idle
+ var onActivityChanged: (@MainActor () -> Void)?
+ var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?
private let provider: LlmProvider
private let assetStore: AssetStore
private let modelAssets: [Asset]
- private var transcriptionTask: Task<Void, Never>?
- private var preloadTask: Task<Void, Never>?
- private var voiceOwner = UUID()
- private var activeVoiceTaskId = UUID()
- private var activeDownloadId: UUID?
- private var activeTaskCount = 0
-
- var hasActiveTasks: Bool { activeTaskCount > 0 }
-
- private let recorder = PcmAudioRecorder()
+ private let usesLiveDictation: Bool
+ private var session: DictationSession?
+ private var stream: DictationStream?
+ private var recorder: LivePcmRecorder?
+ private var task: Task<Void, Never>?
+ private var finishingWatchdog: Task<Void, Never>?
+ private var attemptFinished = false
+ private var insertion: DictationInsertion?
+ private var generation = UUID()
+ private var foreground = true
+ private var awaitingPermission = false
+ private var startOnForeground = false
+ private let notifications = VoiceNotificationTokens()
+ @Published private(set) var hasActiveTasks = false {
+ didSet { onActivityChanged?() }
+ }
+ private func diagnostic(_ event: String) {
+ #if DEBUG
+ EnsuLogging.shared.logger("EnsuVoice").info(event)
+ #endif
+ }
init(provider: LlmProvider, assetStore: AssetStore) {
self.provider = provider
self.assetStore = assetStore
- self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]
+ usesLiveDictation = supportsLiveVoice()
+ modelAssets = [
+ transcriptionModelAsset(),
+ voiceActivityModelAsset(),
+ ]
+ for name in [
+ AVAudioSession.interruptionNotification, AVAudioSession.routeChangeNotification,
+ AVAudioSession.mediaServicesWereResetNotification,
+ ] {
+ notifications.values.append(
+ NotificationCenter.default.addObserver(forName: name, object: nil, queue: .main) {
+ [weak self] notification in
+ let route = notification.name == AVAudioSession.routeChangeNotification
+ let reason = notification.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt
+ Task { @MainActor [weak self] in
+ guard let self,
+ self.active,
+ !route || self.recorder?.isRecording == true || self.state.isRecording
+ else { return }
+ let audio = AVAudioSession.sharedInstance()
+ if route,
+ reason == AVAudioSession.RouteChangeReason.categoryChange.rawValue,
+ audio.category == .playAndRecord, audio.mode == .measurement
+ {
+ return
+ }
+ self.diagnostic(
+ "audio_interruption route=\(route) reason=\(reason ?? 0)")
+ self.interrupt()
+ }
+ })
+ }
}
- private func launchVoiceTask(
- priority: TaskPriority, operation: @escaping @Sendable () async -> Void
- ) -> Task<Void, Never> {
- activeTaskCount += 1
- if activeTaskCount == 1 { onTaskActivityChanged?(true) }
- let task = Task.detached(priority: priority, operation: operation)
- Task { [weak self] in
- await task.value
- guard let self else { return }
- self.activeTaskCount -= 1
- if self.activeTaskCount == 0 { self.onTaskActivityChanged?(false) }
+ func open(insertion: DictationInsertion) {
+ guard !active, !hasActiveTasks else { return }
+ generation = UUID()
+ self.insertion = insertion
+ attemptFinished = false
+ session = DictationSession(live: usesLiveDictation)
+ snapshot = session?.snapshot()
+ active = true
+ state = .idle
+ notice = nil
+ record()
+ }
+
+ func setForeground(_ active: Bool) {
+ foreground = active
+ if active {
+ if startOnForeground {
+ startOnForeground = false
+ record()
+ } else if let owner = session {
+ refresh(owner, token: generation)
+ }
+ } else if !awaitingPermission, !startOnForeground {
+ interrupt()
}
- return task
}
- func startRecording(
- onState: @escaping StateHandler,
- shouldStartRecording: @escaping @MainActor @Sendable () -> Bool = { true }
- ) {
- guard !recorder.isRecording else { return }
+ func composerPreview(original: String) -> DictationInsertionResult? {
+ guard active, let insertion, let snapshot else { return nil }
+ return try? insertion.preview(
+ text: original, voiceText: snapshot.text + snapshot.hypothesis)
+ }
- let session = AVAudioSession.sharedInstance()
- switch session.recordPermission {
- case .granted:
- prepareModelAndStartRecording(
- onState: onState, shouldStartRecording: shouldStartRecording)
- case .denied:
- onState(.error("Microphone permission is required for voice input."))
+ func record() {
+ guard active, foreground, !hasActiveTasks, !attemptFinished, let owner = session
+ else { return }
+ let token = generation
+ switch AVAudioSession.sharedInstance().recordPermission {
case .undetermined:
- session.requestRecordPermission { @Sendable [weak self] granted in
- Task { @MainActor in
- guard let self else { return }
+ guard !awaitingPermission else { return }
+ awaitingPermission = true
+ AVAudioSession.sharedInstance().requestRecordPermission {
+ @Sendable [weak self] granted in
+ Task { @MainActor [weak self] in
+ guard let self, self.generation == token else { return }
+ guard self.active else { return }
+ self.awaitingPermission = false
if granted {
- self.prepareModelAndStartRecording(
- onState: onState,
- shouldStartRecording: shouldStartRecording
- )
+ if self.foreground { self.record() } else { self.startOnForeground = true }
} else {
- onState(.error("Microphone permission is required for voice input."))
+ self.cancel()
+ self.state = .error(
+ "Microphone permission is required. Enable it in Settings.")
}
}
}
- @unknown default:
- onState(.error("Microphone permission is required for voice input."))
- }
- }
-
- func stopAndTranscribe(
- onState: @escaping StateHandler,
- onTranscript: @escaping TranscriptHandler
- ) {
- guard recorder.isRecording else { return }
- onState(.transcribing)
- let recording = recorder.stop()
-
- guard recording.pcm.count >= minimumRecordingBytes(sampleRate: recording.sampleRate) else {
- releaseVoice()
- onState(.error("No speech captured."))
return
+ case .denied:
+ cancel()
+ state = .error("Microphone permission is required. Enable it in Settings.")
+ return
+ case .granted: break
+ @unknown default: return
}
-
- let taskId = beginVoiceTask()
- let downloadId = beginDownload(taskId: taskId)
- let provider = provider
- let sampleRate = recording.sampleRate
- let pcm = recording.pcm
- let owner = voiceOwner
-
- transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
- defer { Task.detached { await provider.releaseVoice(owner: owner) } }
- do {
- try await self?.downloadModelsIfNeeded(
- taskId: taskId,
- downloadId: downloadId,
- onState: onState
- )
-
- if Task.isCancelled { return }
- let isActive = await MainActor.run { [weak self] in
- self?.finishDownload(downloadId: downloadId)
- return self?.isVoiceTaskActive(taskId) == true
+ hasActiveTasks = true
+ notice = nil
+ state = .preparing
+ task = Task { [weak self] in
+ guard let self else { return }
+ defer {
+ self.hasActiveTasks = false
+ if self.generation == token {
+ self.finishingWatchdog?.cancel()
+ self.finishingWatchdog = nil
+ self.recorder?.stop()
+ self.recorder = nil
+ self.stream = nil
+ self.attemptFinished = true
+ self.refresh(owner, token: token)
}
- guard isActive else { return }
- await MainActor.run { [weak self] in
- guard self?.isVoiceTaskActive(taskId) == true else { return }
- onState(.transcribing)
+ }
+ do {
+ if !self.modelAssets.allSatisfy({ self.assetStore.isDownloaded($0) }) {
+ self.state = .downloading(percent: nil)
+ try await self.assetStore.download(assets: self.modelAssets) {
+ [weak self] progress in
+ Task { @MainActor [weak self] in
+ guard let self, self.generation == token,
+ case .downloading = self.state
+ else { return }
+ self.state = .downloading(percent: Int(progress.percentage))
+ }
+ }
}
- let preload = await MainActor.run { self?.preloadTask }
- await preload?.value
try Task.checkCancellation()
- let transcript = try await provider.transcribe(
- inputSampleRate: sampleRate, pcmLe: pcm
- )
- .trimmingCharacters(in: .whitespacesAndNewlines)
-
- if Task.isCancelled { return }
- await MainActor.run { [weak self] in
- guard self?.isVoiceTaskActive(taskId) == true else { return }
- if transcript.isEmpty {
- onState(.error("No speech detected."))
- } else {
- onTranscript(transcript)
- onState(.idle)
+ guard self.generation == token, self.foreground else { return }
+ self.state = .preparing
+ let recorder = try LivePcmRecorder()
+ self.recorder = recorder
+ let sampleRate = recorder.sampleRate
+ let stream = try DictationStream(
+ session: owner, sampleRate: sampleRate, channels: 1)
+ self.stream = stream
+ let delivery = LiveCaptureDelivery(
+ stream: stream, sampleRate: sampleRate,
+ level: { [weak self] level in
+ Task { @MainActor [weak self] in
+ guard let self, self.generation == token, self.state.isRecording else {
+ return
+ }
+ self.audioLevels = Array(self.audioLevels.dropFirst()) + [level]
+ }
}
- }
- } catch is CancellationError {
- return
- } catch {
- if case AssetDownloadError.Cancelled = error { return }
- if case LlmError.Cancelled = error { return }
- await MainActor.run { [weak self] in
- self?.finishDownload(downloadId: downloadId)
- guard self?.isVoiceTaskActive(taskId) == true else { return }
- onState(
- .error(
- error is ModelMemoryUnavailable
- ? error.localizedDescription : "Could not transcribe voice input."))
- }
- }
- }
- }
-
- func cancel() {
- releaseVoice()
- transcriptionTask?.cancel()
- transcriptionTask = nil
- activeVoiceTaskId = UUID()
- activeDownloadId = nil
- if recorder.isRecording {
- _ = recorder.stop()
- }
- }
-
- private func prepareModelAndStartRecording(
- onState: @escaping StateHandler,
- shouldStartRecording: @escaping @MainActor @Sendable () -> Bool
- ) {
- let taskId = beginVoiceTask()
- let downloadId = beginDownload(taskId: taskId)
-
- transcriptionTask = launchVoiceTask(priority: .userInitiated) { [weak self] in
- do {
- try await self?.downloadModelsIfNeeded(
- taskId: taskId,
- downloadId: downloadId,
- onState: onState
- )
-
- if Task.isCancelled { return }
- await MainActor.run { [weak self] in
- self?.finishDownload(downloadId: downloadId)
- guard let self, self.isVoiceTaskActive(taskId) else { return }
- guard shouldStartRecording() else {
- onState(.idle)
- return
+ ) {
+ [weak self] in
+ Task { @MainActor [weak self] in
+ guard let self, self.generation == token else { return }
+ self.interrupt(
+ message:
+ "Audio capture interrupted. Some words may be missing; review the text."
+ )
}
- self.beginRecording(onState: onState)
- if self.recorder.isRecording { self.preloadTranscriptionModel() }
}
+ let callback = LiveVoiceCallback(
+ ready: { [weak self] in
+ Task { @MainActor [weak self] in
+ let value = owner.snapshot()
+ guard let self, self.generation == token, self.foreground,
+ self.active, self.stream === stream, value.workerActive,
+ value.phase == .listening
+ else { return }
+ do {
+ try recorder.start(delivery: delivery)
+ self.state = .recording
+ self.audioLevels = Array(repeating: 0, count: 24)
+ self.notice = nil
+ self.refresh(owner, token: token)
+ } catch {
+ self.interrupt(
+ message:
+ "Could not start the microphone. Tap the microphone to start again."
+ )
+ }
+ }
+ },
+ change: { [weak self] in
+ Task { @MainActor [weak self] in self?.refresh(owner, token: token) }
+ },
+ closed: { [weak self] in
+ delivery.close()
+ Task { @MainActor [weak self] in
+ guard let self, self.generation == token, self.stream === stream
+ else { return }
+ recorder.stop()
+ self.watchFinalization(owner, stream: stream, token: token)
+ }
+ }
+ )
+ try await self.provider.runLiveVoice(stream: stream, callback: callback)
} catch is CancellationError {
- return
} catch {
- if case AssetDownloadError.Cancelled = error { return }
- if case LlmError.Cancelled = error { return }
- await MainActor.run { [weak self] in
- self?.finishDownload(downloadId: downloadId)
- guard self?.isVoiceTaskActive(taskId) == true else { return }
- onState(.error("Voice model download failed."))
+ if let error = error as? DictationError, case .InvalidOperation(let detail) = error
+ {
+ self.diagnostic("failure \(detail)")
+ } else {
+ self.diagnostic("failure \(type(of: error))")
}
- }
- }
- }
-
- private nonisolated func downloadModelsIfNeeded(
- taskId: UUID,
- downloadId: UUID,
- onState: @escaping StateHandler
- ) async throws {
- let assets = modelAssets
- let assetStore = self.assetStore
- if assets.allSatisfy({ assetStore.isDownloaded($0) }) {
- return
- }
- await MainActor.run { [weak self] in
- guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else {
- return
- }
- onState(.downloading(percent: nil))
- }
- try await assetStore.download(assets: assets) { [weak self] progress in
- let percent = min(max(Int(progress.percentage), 0), 100)
- Task { @MainActor [weak self] in
- guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else {
- return
+ if self.generation == token, self.notice == nil {
+ self.notice = "Voice input stopped. Review the available text or try again."
}
- onState(.downloading(percent: percent))
}
}
}
- private func beginVoiceTask() -> UUID {
- transcriptionTask?.cancel()
- let taskId = UUID()
- activeVoiceTaskId = taskId
- activeDownloadId = nil
- return taskId
- }
-
- private func preloadTranscriptionModel() {
- let provider = provider
- voiceOwner = UUID()
- let owner = voiceOwner
- preloadTask = launchVoiceTask(priority: .utility) {
- try? await provider.prewarmVoice(owner: owner)
+ private func refresh(_ owner: DictationSession, token: UUID) {
+ guard generation == token, active else { return }
+ let value = owner.snapshot()
+ snapshot = value
+ guard attemptFinished else {
+ if value.phase == .finishing { state = .transcribing }
+ return
}
- }
-
- private func releaseVoice() {
- preloadTask?.cancel()
- preloadTask = nil
- let owner = voiceOwner
- voiceOwner = UUID()
- let provider = provider
- _ = launchVoiceTask(priority: .utility) { await provider.releaseVoice(owner: owner) }
- }
-
- private func isVoiceTaskActive(_ taskId: UUID) -> Bool {
- activeVoiceTaskId == taskId
- }
-
- private func beginDownload(taskId: UUID) -> UUID {
- let downloadId = UUID()
- if isVoiceTaskActive(taskId) {
- activeDownloadId = downloadId
+ guard foreground else { return }
+ let text = value.text + value.hypothesis
+ let hasText = !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ let added = hasText && insertion.map { onInsert?($0, text) == true } == true
+ let completedState: VoiceInputState
+ if added {
+ completedState = value.incomplete ? .error("Voice input stopped.") : .idle
+ } else {
+ completedState = .error(
+ hasText ? "Voice input stopped." : notice ?? "No speech detected.")
}
- return downloadId
+ active = false
+ session = nil
+ snapshot = nil
+ insertion = nil
+ onInsert = nil
+ audioLevels = Array(repeating: 0, count: 24)
+ state = completedState
}
- private func finishDownload(downloadId: UUID) {
- if activeDownloadId == downloadId {
- activeDownloadId = nil
+ private func watchFinalization(_ owner: DictationSession, stream: DictationStream, token: UUID)
+ {
+ guard token == generation, active, self.stream === stream, owner.snapshot().workerActive,
+ finishingWatchdog == nil
+ else { return }
+ finishingWatchdog = Task { @MainActor [weak self] in
+ do { try await Task.sleep(for: .seconds(10)) } catch { return }
+ guard let self, self.generation == token, self.active, self.stream === stream,
+ owner.snapshot().workerActive
+ else { return }
+ self.recorder?.stop()
+ try? stream.abandon(reason: .failure)
+ self.task?.cancel()
+ self.snapshot = owner.snapshot()
+ self.notice =
+ "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."
}
}
- private func isDownloadActive(taskId: UUID, downloadId: UUID) -> Bool {
- isVoiceTaskActive(taskId) && activeDownloadId == downloadId
+ func stop() {
+ guard let owner = session else { return }
+ let reason: DictationStopReason = .done
+ _ = try? owner.stop(reason: reason)
+ snapshot = owner.snapshot()
+ state = .transcribing
+ let stream = stream
+ if let stream { watchFinalization(owner, stream: stream, token: generation) }
+ recorder?.stop(afterDelivery: { try? stream?.stop(reason: reason) })
+ if recorder == nil { try? stream?.stop(reason: reason) }
}
- private func beginRecording(onState: @escaping StateHandler) {
- do {
- try recorder.start()
- onState(.recording)
- } catch {
- onState(.error("Could not record microphone audio."))
+ func interrupt(message: String = "Voice input stopped.") {
+ guard active, !attemptFinished else { return }
+ if !hasActiveTasks, stream == nil {
+ cancel()
+ return
}
+ recorder?.stop()
+ try? stream?.abandon(reason: .interruption)
+ task?.cancel()
+ notice = message
+ state = .transcribing
}
- private func minimumRecordingBytes(sampleRate: UInt32) -> Int {
- Int(sampleRate) / 4 * 2
+ func cancel() {
+ awaitingPermission = false
+ startOnForeground = false
+ generation = UUID()
+ finishingWatchdog?.cancel()
+ finishingWatchdog = nil
+ recorder?.stop()
+ try? stream?.abandon(reason: .discard)
+ task?.cancel()
+ active = false
+ state = .idle
+ snapshot = nil
+ session = nil
+ insertion = nil
+ stream = nil
+ recorder = nil
+ onInsert = nil
}
}
-private struct VoiceRecording {
- let sampleRate: UInt32
- let pcm: Data
+private final class LiveVoiceCallback: DictationStreamCallback, Sendable {
+ let ready: @Sendable () -> Void
+ let change: @Sendable () -> Void
+ let closed: @Sendable () -> Void
+ init(
+ ready: @escaping @Sendable () -> Void,
+ change: @escaping @Sendable () -> Void,
+ closed: @escaping @Sendable () -> Void
+ ) {
+ self.ready = ready; self.change = change; self.closed = closed
+ }
+ func onReady() { ready() }
+ func onChange() { change() }
+ func onCaptureClosed() { closed() }
}
-private final class PcmAudioRecorder {
+@MainActor
+private final class LivePcmRecorder {
private let engine = AVAudioEngine()
- private let lock = NSLock()
- private var pcm = Data()
- private var sampleRate: UInt32 = 16_000
-
- var isRecording: Bool {
- engine.isRunning
- }
-
- func start() throws {
- lock.lock()
- pcm.removeAll(keepingCapacity: true)
- lock.unlock()
+ private var delivery: LiveCaptureDelivery?
+ private var installed = false
+ let sampleRate: UInt32
+ var isRecording: Bool { engine.isRunning }
- let session = AVAudioSession.sharedInstance()
- try session.setCategory(
+ init() throws {
+ let audio = AVAudioSession.sharedInstance()
+ try audio.setCategory(
.playAndRecord, mode: .measurement, options: [.allowBluetoothHFP, .defaultToSpeaker])
- try session.setActive(true)
+ try audio.setActive(true)
+ sampleRate = UInt32(engine.inputNode.outputFormat(forBus: 0).sampleRate.rounded())
+ guard (8000...48000).contains(sampleRate) else {
+ try? audio.setActive(false, options: .notifyOthersOnDeactivation)
+ throw CocoaError(.featureUnsupported)
+ }
+ }
+ func start(delivery: LiveCaptureDelivery) throws {
let input = engine.inputNode
let format = input.outputFormat(forBus: 0)
- sampleRate = UInt32(format.sampleRate.rounded())
-
- input.removeTap(onBus: 0)
- input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak self] buffer, _ in
- self?.append(buffer)
+ guard UInt32(format.sampleRate.rounded()) == sampleRate else {
+ throw CocoaError(.featureUnsupported)
}
-
+ self.delivery = delivery
+ input.installTap(onBus: 0, bufferSize: AVAudioFrameCount(sampleRate / 10), format: format) {
+ @Sendable buffer, _ in
+ delivery.accept(buffer)
+ }
+ installed = true
engine.prepare()
try engine.start()
}
- func stop() -> VoiceRecording {
- engine.inputNode.removeTap(onBus: 0)
+ func stop(afterDelivery: (@Sendable () -> Void)? = nil) {
+ if installed { engine.inputNode.removeTap(onBus: 0); installed = false }
+ delivery?.close()
engine.stop()
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
-
- lock.lock()
- let result = VoiceRecording(sampleRate: sampleRate, pcm: pcm)
- pcm.removeAll(keepingCapacity: true)
- lock.unlock()
- return result
+ if let afterDelivery {
+ if let delivery { delivery.drain(afterDelivery) } else { afterDelivery() }
+ }
}
+}
- private func append(_ buffer: AVAudioPCMBuffer) {
- guard let channels = buffer.floatChannelData else { return }
- let frameCount = Int(buffer.frameLength)
- let channelCount = max(Int(buffer.format.channelCount), 1)
- guard frameCount > 0 else { return }
+private final class VoiceNotificationTokens: @unchecked Sendable {
+ var values: [NSObjectProtocol] = []
+ deinit { for value in values { NotificationCenter.default.removeObserver(value) } }
+}
- var chunk = Data(capacity: frameCount * 2)
- for frame in 0..<frameCount {
- var sample: Float = 0
- for channel in 0..<channelCount {
- sample += channels[channel][frame]
- }
- sample /= Float(channelCount)
- let clamped = min(max(sample, -1), 1)
- var intSample = Int16(clamped * Float(Int16.max)).littleEndian
- withUnsafeBytes(of: &intSample) { bytes in
- chunk.append(contentsOf: bytes)
- }
- }
+private final class LiveCaptureDelivery: @unchecked Sendable {
+ private let queue = DispatchQueue(label: "io.ente.ensu.voice.capture", qos: .userInitiated)
+ private let lock = NSLock()
+ private var accepting = true
+ private var framesPending = 0
+ private var chunksPending = 0
+ private let stream: DictationStream
+ private let rate: Int
+ private let interrupted: @Sendable () -> Void
+ private let level: @Sendable (Float) -> Void
+ private var meterFrames = 0
+ private var meterEnergy = 0.0
+
+ init(
+ stream: DictationStream, sampleRate: UInt32,
+ level: @escaping @Sendable (Float) -> Void, interrupted: @escaping @Sendable () -> Void
+ ) {
+ self.stream = stream; rate = Int(sampleRate); self.interrupted = interrupted
+ self.level = level
+ }
+ func close() { lock.lock(); accepting = false; lock.unlock() }
+ func drain(_ completion: @escaping @Sendable () -> Void) { queue.async(execute: completion) }
+ func accept(_ buffer: AVAudioPCMBuffer) {
+ let frames = Int(buffer.frameLength)
+ guard frames > 0 else { return }
lock.lock()
- pcm.append(chunk)
+ guard accepting else { lock.unlock(); return }
+ guard frames <= rate, framesPending + frames <= rate * 2, chunksPending < 128,
+ let channels = buffer.floatChannelData, buffer.format.channelCount > 0
+ else {
+ accepting = false
+ lock.unlock()
+ queue.async { [self] in
+ try? stream.abandon(reason: .failure); interrupted()
+ }
+ return
+ }
+ framesPending += frames
+ chunksPending += 1
lock.unlock()
+ let channelCount = Int(buffer.format.channelCount)
+ var mono = [Float](repeating: 0, count: frames)
+ for frame in 0..<frames {
+ for channel in 0..<channelCount { mono[frame] += channels[channel][frame] }
+ mono[frame] /= Float(channelCount)
+ }
+ let samples = mono
+ queue.async { [self] in
+ defer {
+ lock.lock(); framesPending -= frames; chunksPending -= 1; lock.unlock()
+ }
+ do {
+ meterFrames += frames
+ meterEnergy += samples.reduce(0.0) { $0 + Double($1 * $1) }
+ if meterFrames >= rate / 10 {
+ let rms = sqrt(meterEnergy / Double(meterFrames))
+ level(Float(min(1, max(0, (20 * log10(max(rms, 0.00001)) + 55) / 45))))
+ meterFrames = 0; meterEnergy = 0
+ }
+ for start in stride(from: 0, to: frames, by: rate / 10) {
+ let end = min(frames, start + rate / 10)
+ var pcm = Data(count: (end - start) * 2)
+ pcm.withUnsafeMutableBytes { raw in
+ let values = raw.bindMemory(to: Int16.self)
+ for index in start..<end {
+ let value = samples[index]
+ values[index - start] =
+ Int16(min(max(value.isFinite ? value : 0, -1), 1) * 32767)
+ .littleEndian
+ }
+ }
+ try stream.feed(pcmLe: pcm)
+ }
+ } catch {
+ lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()
+ if wasAccepting { try? stream.abandon(reason: .failure); interrupted() }
+ }
+ }
}
}
diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index 2c308bd5e7..aa73e87cd8 100644
--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -118,7 +118,6 @@ actor LlmProvider {
private var currentContextLength: Int?
private var backendInitialized = false
private var chatWarmupOwner: UUID?
- private var voiceOwner: UUID?
private var chatForeground = false
private var appForeground = false
private var memoryPressure = ModelMemoryPressureState()
@@ -169,51 +168,38 @@ actor LlmProvider {
appForeground: appForeground, now: ProcessInfo.processInfo.systemUptime
)
else { return }
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
unloadModel()
memoryPressure.didCleanUp()
}
- func prewarmVoice(owner: UUID) async throws {
- try await withModelLock {
- try Task.checkCancellation()
- guard chatForeground, !memoryPressureSuppressed else { return }
- try prepareVoice()
- do {
- let transcriber = transcriber
- try await Task.detached { try transcriber.loadModel() }.value
- try Task.checkCancellation()
- voiceOwner = owner
- } catch {
- unloadTranscriptionModelIfLoaded()
- throw error
+ func runLiveVoice(
+ stream: DictationStream, callback: any DictationStreamCallback
+ ) async throws {
+ do {
+ try await withTaskCancellationHandler {
+ try await withModelLock {
+ try Task.checkCancellation()
+ try prepareVoice()
+ let transcriber = transcriber
+ defer { transcriber.unloadModel() }
+ try await Task.detached(priority: .userInitiated) {
+ try stream.run(transcriber: transcriber, callback: callback)
+ }.value
+ try Task.checkCancellation()
+ }
+ } onCancel: {
+ try? stream.abandon(reason: .interruption)
}
- }
- }
-
- func releaseVoice(owner: UUID) async {
- try? await modelLoadGate.withLock {
- if voiceOwner == owner { unloadTranscriptionModelIfLoaded() }
- }
- }
-
- func transcribe(inputSampleRate: UInt32, pcmLe: Data) async throws -> String {
- try await withModelLock {
- try Task.checkCancellation()
- try prepareVoice()
- defer { unloadTranscriptionModelIfLoaded() }
- let transcriber = transcriber
- let result = try await Task.detached {
- try transcriber.transcribe(inputSampleRate: inputSampleRate, pcmLe: pcmLe)
- }.value
- try Task.checkCancellation()
- return result
+ } catch {
+ try? stream.abandon(reason: .failure)
+ throw error
}
}
private func prepareVoice() throws {
loadedContext?.releaseMultimodal()
- if voiceOwner == nil && !canAllocate(.voice) {
+ if !canAllocate(.voice) {
unloadModel()
guard hasRequiredWorkReserve() else { throw ModelMemoryUnavailable() }
}
@@ -355,14 +341,14 @@ actor LlmProvider {
_ selection: LlmModelSelection, owner: UUID
) async throws -> ChatWarmupResult {
let result: ChatWarmupResult = try await withModelLock(retainAfterCooldown: true) {
- guard appForeground, chatForeground, voiceOwner == nil,
- isChatModelReady(selection)
- else { return .skipped }
+ guard appForeground, chatForeground, isChatModelReady(selection) else {
+ return .skipped
+ }
let cooldown = memoryPressure.remainingCooldown(
now: ProcessInfo.processInfo.systemUptime)
if cooldown > 0 { return .retryAfter(cooldown) }
guard !memoryPressureSuppressed else { return .skipped }
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
try Task.checkCancellation()
guard loadedContextLength(selection) == nil else {
if chatWarmupOwner != nil { chatWarmupOwner = owner }
@@ -410,7 +396,7 @@ actor LlmProvider {
return
}
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
unloadModel()
if !backendInitialized {
@@ -467,11 +453,11 @@ actor LlmProvider {
try await withTaskCancellationHandler {
try await withModelLock {
try control.checkCancellation()
- guard chatForeground, !memoryPressureSuppressed, voiceOwner == nil,
+ guard chatForeground, !memoryPressureSuppressed,
loadedContextLength(selection) != nil,
let model = loadedModel, let loadedContext
else { throw ModelMemoryDeferred() }
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
loadedContext.releaseMultimodal()
guard canAllocate(.title) else { throw ModelMemoryDeferred() }
try control.checkCancellation()
@@ -522,7 +508,7 @@ actor LlmProvider {
guard let context = loadedContext else {
throw CancellationError()
}
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
try control.checkCancellation()
return try await operation(context, control)
}
@@ -649,10 +635,10 @@ actor LlmProvider {
throw EmbeddingAssetInvalidError()
}
- if maintenance && (voiceOwner != nil || memoryPressureSuppressed) {
+ if maintenance && memoryPressureSuppressed {
throw ModelMemoryDeferred()
}
- unloadTranscriptionModelIfLoaded()
+ transcriber.unloadModel()
loadedContext?.releaseMultimodal()
if !canAllocate(.embedding, queryBytes: maintenance ? nil : queryBytes) {
if maintenance && chatForeground && loadedModel != nil {
@@ -736,11 +722,6 @@ actor LlmProvider {
currentContextLength = nil
}
- private func unloadTranscriptionModelIfLoaded() {
- voiceOwner = nil
- transcriber.unloadModel()
- }
-
private nonisolated func chatAsset(_ selection: LlmModelSelection) -> Asset {
llmAsset(modelId: selection.id)
}
diff --git a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
index 23b45c5b68..44292bdc0d 100644
--- a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
+++ b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
@@ -26,6 +26,10 @@ enum ChatDeviceCapability: Equatable {
}
}
+func supportsLiveVoice(totalMemoryBytes: UInt64 = ProcessInfo.processInfo.physicalMemory) -> Bool {
+ totalMemoryBytes > 4 * 1024 * 1024 * 1024
+}
+
func currentChatDeviceCapability() -> ChatDeviceCapability {
chatDeviceCapability(totalMemoryBytes: ProcessInfo.processInfo.physicalMemory)
}
diff --git a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
index cc3ad618bd..2765c1b472 100644
--- a/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
+++ b/apple/apps/ensu/EnsuTests/ModelMemoryCoordinationTests.swift
@@ -20,6 +20,16 @@ private actor ModelOperationBarrier {
@MainActor
final class ModelMemoryCoordinationTests: XCTestCase {
+ func testLiveVoiceRequiresMoreThanFourGiB() {
+ let limit: UInt64 = 4 * 1024 * 1024 * 1024
+ for bytes: UInt64 in [0, 4_000_000_000, limit] {
+ XCTAssertFalse(supportsLiveVoice(totalMemoryBytes: bytes))
+ }
+ for bytes: UInt64 in [limit + 1, 8_000_000_000] {
+ XCTAssertTrue(supportsLiveVoice(totalMemoryBytes: bytes))
+ }
+ }
+
func testMinimumRamBlocksOnlyKnownReadingsBelowTheFloor() {
XCTAssertFalse(chatDeviceCapability(totalMemoryBytes: 1_599_999_999).isChatSupported)
XCTAssertTrue(chatDeviceCapability(totalMemoryBytes: 1_600_000_000).isChatSupported)
diff --git a/rust/bindings/uniffi/ensu/src/conversation.rs b/rust/bindings/uniffi/ensu/src/conversation.rs
index a13f7bd255..07a60c6368 100644
--- a/rust/bindings/uniffi/ensu/src/conversation.rs
+++ b/rust/bindings/uniffi/ensu/src/conversation.rs
@@ -17,6 +17,10 @@ pub enum ConversationError {
Cancelled,
#[error("Conversation changed; retry the reply")]
Stale,
+ #[error(
+ "This message is too long for the model, even without earlier messages. Shorten it or split it into smaller messages. Your full text is still available to edit or copy."
+ )]
+ CurrentInputTooLarge,
#[error("{detail}")]
Other { detail: String },
}
@@ -32,6 +36,7 @@ impl From<core::PrepareError> for ConversationError {
match cause {
core::PrepareError::Cancelled => Self::Cancelled,
core::PrepareError::Stale => Self::Stale,
+ core::PrepareError::CurrentInputTooLarge => Self::CurrentInputTooLarge,
cause => error(cause),
}
}
diff --git a/rust/bindings/uniffi/ensu/src/lib.rs b/rust/bindings/uniffi/ensu/src/lib.rs
index 917d5f5a46..b7386167fb 100644
--- a/rust/bindings/uniffi/ensu/src/lib.rs
+++ b/rust/bindings/uniffi/ensu/src/lib.rs
@@ -11,3 +11,4 @@ pub mod log;
pub mod notes;
pub mod retrieval;
pub mod transcription;
+pub mod voice;
diff --git a/rust/bindings/uniffi/ensu/src/transcription.rs b/rust/bindings/uniffi/ensu/src/transcription.rs
index f149f83b5b..5163b6f068 100644
--- a/rust/bindings/uniffi/ensu/src/transcription.rs
+++ b/rust/bindings/uniffi/ensu/src/transcription.rs
@@ -41,7 +41,7 @@ impl From<transcription::TranscriptionError> for TranscriptionError {
#[derive(uniffi::Object)]
pub struct Transcriber {
- inner: transcription::Transcriber,
+ pub(crate) inner: transcription::Transcriber,
}
#[uniffi::export]
@@ -53,6 +53,23 @@ impl Transcriber {
})
}
+ #[uniffi::constructor]
+ pub fn with_cache(
+ model_dir: String,
+ vad_model_path: String,
+ cache_directory: String,
+ device_identity: String,
+ ) -> Arc<Self> {
+ Arc::new(Self {
+ inner: transcription::Transcriber::with_cache(
+ model_dir,
+ vad_model_path,
+ cache_directory,
+ device_identity,
+ ),
+ })
+ }
+
pub fn load_model(&self) -> Result<(), TranscriptionError> {
self.inner.load_model().map_err(Into::into)
}
diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rs
new file mode 100644
index 0000000000..ba3f5cb152
--- /dev/null
+++ b/rust/bindings/uniffi/ensu/src/voice.rs
@@ -0,0 +1,497 @@
+use ente_ensu::transcription::{DictationState, LiveControl, LiveEvent, LiveTranscription};
+use ente_ensu::voice;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
+use std::time::{Duration, Instant};
+
+#[derive(Debug, thiserror::Error, uniffi::Error)]
+pub enum DictationError {
+ #[error("{detail}")]
+ InvalidOperation { detail: String },
+}
+
+impl From<voice::VoiceError> for DictationError {
+ fn from(value: voice::VoiceError) -> Self {
+ Self::InvalidOperation {
+ detail: value.to_string(),
+ }
+ }
+}
+
+#[derive(Debug, Clone, uniffi::Record)]
+pub struct DictationInsertionResult {
+ pub text: String,
+ pub revision: u64,
+ pub caret_utf16: u32,
+}
+
+#[derive(uniffi::Object)]
+pub struct DictationInsertion {
+ anchor: voice::ComposerAnchor,
+ transfer: Mutex<voice::ComposerTransfer>,
+}
+
+#[uniffi::export]
+impl DictationInsertion {
+ #[uniffi::constructor]
+ pub fn new(destination: String, revision: u64, start_utf16: u32, end_utf16: u32) -> Arc<Self> {
+ Arc::new(Self {
+ anchor: voice::ComposerAnchor {
+ destination,
+ revision,
+ start_utf16,
+ end_utf16,
+ },
+ transfer: Mutex::new(voice::ComposerTransfer::default()),
+ })
+ }
+
+ pub fn preview(
+ &self,
+ text: String,
+ voice_text: String,
+ ) -> Result<DictationInsertionResult, DictationError> {
+ let (text, caret_utf16) = voice::preview_insertion(&text, &self.anchor, &voice_text)?;
+ Ok(DictationInsertionResult {
+ text,
+ revision: self.anchor.revision,
+ caret_utf16,
+ })
+ }
+
+ pub fn apply(
+ &self,
+ destination: String,
+ mut revision: u64,
+ mut text: String,
+ voice_text: String,
+ foreground: bool,
+ keyboard_composing: bool,
+ ) -> Result<DictationInsertionResult, DictationError> {
+ let caret_utf16 = self
+ .transfer
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .apply(
+ &voice::VoiceInsertion {
+ anchor: self.anchor.clone(),
+ text: voice_text,
+ },
+ &destination,
+ &mut revision,
+ &mut text,
+ foreground,
+ keyboard_composing,
+ )?;
+ Ok(DictationInsertionResult {
+ text,
+ revision,
+ caret_utf16,
+ })
+ }
+}
+
+#[derive(Debug, Clone, Copy, uniffi::Enum)]
+pub enum DictationPhase {
+ Preparing,
+ Listening,
+ Countdown,
+ Finishing,
+ Ready,
+ Closed,
+}
+
+#[derive(Debug, Clone, Copy, uniffi::Enum)]
+pub enum DictationStopReason {
+ Done,
+ Interruption,
+ Failure,
+ Discard,
+}
+impl From<DictationStopReason> for voice::StopReason {
+ fn from(value: DictationStopReason) -> Self {
+ match value {
+ DictationStopReason::Done => Self::Done,
+ DictationStopReason::Interruption => Self::Interruption,
+ DictationStopReason::Failure => Self::Failure,
+ DictationStopReason::Discard => Self::Discard,
+ }
+ }
+}
+#[derive(Debug, Clone, uniffi::Record)]
+pub struct DictationSnapshot {
+ pub phase: DictationPhase,
+ pub text: String,
+ pub hypothesis: String,
+ pub worker_active: bool,
+ pub incomplete: bool,
+}
+#[derive(uniffi::Object)]
+pub struct DictationSession {
+ state: Mutex<DictationState>,
+ live: bool,
+}
+
+#[uniffi::export(foreign)]
+pub trait DictationStreamCallback: Send + Sync {
+ fn on_ready(&self);
+ fn on_change(&self);
+ fn on_capture_closed(&self);
+}
+
+#[derive(uniffi::Object)]
+pub struct DictationStream {
+ session: Arc<DictationSession>,
+ stream: LiveTranscription,
+ started: AtomicBool,
+}
+#[uniffi::export]
+impl DictationStream {
+ #[uniffi::constructor]
+ pub fn new(
+ session: Arc<DictationSession>,
+ sample_rate: u32,
+ channels: u16,
+ ) -> Result<Arc<Self>, DictationError> {
+ let stream = LiveTranscription::new(voice::CaptureFormat {
+ sample_rate,
+ channels,
+ })?;
+ session.lock().session.begin_capture()?;
+ Ok(Arc::new(Self {
+ session,
+ stream,
+ started: AtomicBool::new(false),
+ }))
+ }
+ pub fn feed(&self, pcm_le: &[u8]) -> Result<(), DictationError> {
+ self.stream.feed(pcm_le).map_err(Into::into)
+ }
+ pub fn stop(&self, reason: DictationStopReason) -> Result<(), DictationError> {
+ self.session.lock().session.stop(reason.into())?;
+ if matches!(
+ reason,
+ DictationStopReason::Discard
+ | DictationStopReason::Interruption
+ | DictationStopReason::Failure
+ ) {
+ self.stream.cancel();
+ } else {
+ self.stream.finish();
+ }
+ Ok(())
+ }
+ pub fn abandon(&self, reason: DictationStopReason) -> Result<(), DictationError> {
+ if !matches!(
+ reason,
+ DictationStopReason::Failure
+ | DictationStopReason::Interruption
+ | DictationStopReason::Discard
+ ) {
+ return Err(voice::VoiceError::InvalidState.into());
+ }
+ self.stop(reason)?;
+ if self
+ .started
+ .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+ .is_ok()
+ {
+ self.session.lock().session.worker_exited(false)?;
+ }
+ Ok(())
+ }
+ pub fn run(
+ &self,
+ transcriber: Arc<crate::transcription::Transcriber>,
+ callback: Arc<dyn DictationStreamCallback>,
+ ) -> Result<(), DictationError> {
+ if self.started.swap(true, Ordering::AcqRel) {
+ return Err(voice::VoiceError::InvalidState.into());
+ }
+ let mut notified_stop = false;
+ let mut last_notification = Instant::now();
+ let mut on_event = |event: LiveEvent| {
+ let immediate = matches!(
+ &event,
+ LiveEvent::Commit { .. }
+ | LiveEvent::ReplaceTranscript(_)
+ | LiveEvent::Deferred
+ | LiveEvent::CaptureClosed
+ );
+ if matches!(&event, LiveEvent::Ready) {
+ let phase = self.session.lock().session.phase();
+ if phase == voice::VoicePhase::Closed {
+ return Ok(LiveControl::Cancel);
+ }
+ if phase == voice::VoicePhase::Listening {
+ callback.on_ready();
+ }
+ return Ok(LiveControl::Continue);
+ }
+ let control = self
+ .session
+ .lock()
+ .apply(event, || self.stream.finish_if_empty())?;
+ if immediate && !notified_stop && self.session.lock().session.stop_reason().is_some() {
+ notified_stop = true;
+ callback.on_capture_closed();
+ }
+ if immediate || last_notification.elapsed() >= Duration::from_millis(250) {
+ callback.on_change();
+ last_notification = Instant::now();
+ }
+ Ok(control)
+ };
+ let result = if self.session.live {
+ self.stream.run(&transcriber.inner, &mut on_event)
+ } else {
+ self.stream.run_recorded(&transcriber.inner, &mut on_event)
+ };
+ let mut state = self.session.lock();
+ if result.is_err() && state.session.phase() != voice::VoicePhase::Closed {
+ if state.session.stop_reason() != Some(voice::StopReason::Interruption) {
+ state.session.stop(voice::StopReason::Failure)?;
+ }
+ let accepted = state.session.accepted_samples();
+ let submitted = self.stream.submitted_samples();
+ if submitted > accepted {
+ state.session.observe_drained_audio(accepted, submitted)?;
+ }
+ }
+ state.session.worker_exited(result.is_ok())?;
+ drop(state);
+ self.stream.finish();
+ if !notified_stop {
+ callback.on_capture_closed();
+ }
+ callback.on_change();
+ result.map_err(Into::into)
+ }
+}
+impl DictationSession {
+ fn lock(&self) -> MutexGuard<'_, DictationState> {
+ self.state.lock().unwrap_or_else(PoisonError::into_inner)
+ }
+}
+#[uniffi::export]
+impl DictationSession {
+ #[uniffi::constructor]
+ pub fn new(live: bool) -> Arc<Self> {
+ Arc::new(Self {
+ live,
+ state: Mutex::new(DictationState::new(live)),
+ })
+ }
+ pub fn stop(&self, reason: DictationStopReason) -> Result<bool, DictationError> {
+ self.lock().session.stop(reason.into()).map_err(Into::into)
+ }
+ pub fn snapshot(&self) -> DictationSnapshot {
+ let state = self.lock();
+ let phase = match state.session.phase() {
+ voice::VoicePhase::Preparing => DictationPhase::Preparing,
+ voice::VoicePhase::Listening => DictationPhase::Listening,
+ voice::VoicePhase::Countdown => DictationPhase::Countdown,
+ voice::VoicePhase::Finishing => DictationPhase::Finishing,
+ voice::VoicePhase::Ready => DictationPhase::Ready,
+ voice::VoicePhase::Closed => DictationPhase::Closed,
+ };
+ DictationSnapshot {
+ phase,
+ text: state.text().to_owned(),
+ hypothesis: state.hypothesis(),
+ worker_active: state.session.worker_active(),
+ incomplete: state.session.incomplete(),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[derive(Default)]
+ struct StreamSignals {
+ stopped: std::sync::atomic::AtomicUsize,
+ ready: std::sync::atomic::AtomicUsize,
+ }
+ impl DictationStreamCallback for StreamSignals {
+ fn on_ready(&self) {
+ self.ready.fetch_add(1, Ordering::Relaxed);
+ }
+ fn on_change(&self) {}
+ fn on_capture_closed(&self) {
+ self.stopped.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+
+ #[test]
+ fn composer_previews_replace_the_selection_without_consuming_the_transfer() {
+ let insertion = DictationInsertion::new("chat".into(), 7, 7, 13);
+ let original = " meet Friday\n".to_owned();
+ assert_eq!(
+ insertion
+ .preview(original.clone(), String::new())
+ .unwrap()
+ .text,
+ original
+ );
+ assert_eq!(
+ insertion
+ .preview(original.clone(), "Sat".into())
+ .unwrap()
+ .text,
+ " meet Sat\n"
+ );
+ let preview = insertion
+ .preview(original.clone(), "Saturday".into())
+ .unwrap();
+ assert_eq!(preview.revision, 7);
+ let final_text = insertion
+ .apply("chat".into(), 7, original, "Saturday".into(), true, false)
+ .unwrap();
+ assert_eq!(preview.text, final_text.text);
+ assert_eq!(preview.caret_utf16, final_text.caret_utf16);
+ assert_eq!(final_text.revision, 8);
+ assert!(
+ insertion
+ .apply(
+ "chat".into(),
+ 8,
+ final_text.text,
+ "Sunday".into(),
+ true,
+ false
+ )
+ .is_err()
+ );
+ }
+
+ fn event(session: &DictationSession, event: LiveEvent) {
+ session.lock().apply(event, || true).unwrap();
+ }
+
+ #[test]
+ fn interruption_keeps_visible_text_until_actual_worker_exit() {
+ let owner = DictationSession::new(true);
+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
+ stream.started.store(true, Ordering::Release);
+ event(
+ &owner,
+ LiveEvent::Audio {
+ start: 0,
+ end: 32000,
+ voiced: true,
+ },
+ );
+ event(
+ &owner,
+ LiveEvent::Commit {
+ start: 0,
+ end: 16000,
+ text: "Keep this".into(),
+ },
+ );
+ event(&owner, LiveEvent::Hypothesis("opening word".into()));
+ stream.stop(DictationStopReason::Done).unwrap();
+ stream.abandon(DictationStopReason::Interruption).unwrap();
+ assert!(owner.snapshot().worker_active);
+ assert!(DictationStream::new(owner.clone(), 16000, 1).is_err());
+ owner.lock().session.worker_exited(false).unwrap();
+ let value = owner.snapshot();
+ assert_eq!(
+ format!("{}{}", value.text, value.hypothesis),
+ "Keep this opening word"
+ );
+ assert!(!value.worker_active);
+ assert!(value.incomplete);
+ assert!(matches!(value.phase, DictationPhase::Ready));
+ let insertion = DictationInsertion::new("chat".into(), 0, 7, 7);
+ let draft = insertion
+ .apply(
+ "chat".into(),
+ 0,
+ "Notes: ".into(),
+ value.text + &value.hypothesis,
+ true,
+ false,
+ )
+ .unwrap();
+ assert_eq!(draft.text, "Notes: Keep this opening word");
+ assert!(matches!(
+ owner
+ .lock()
+ .apply(LiveEvent::Hypothesis("late".into()), || true)
+ .unwrap(),
+ LiveControl::Cancel
+ ));
+ assert!(DictationStream::new(owner, 16000, 1).is_err());
+ assert!(DictationStream::new(DictationSession::new(true), 16000, 1).is_ok());
+ }
+ #[test]
+ fn format_failure_does_not_reserve_a_worker_and_cancel_before_run_cannot_resume() {
+ let owner = DictationSession::new(true);
+ assert!(DictationStream::new(owner.clone(), 0, 1).is_err());
+ assert!(!owner.snapshot().worker_active);
+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
+ stream.abandon(DictationStopReason::Failure).unwrap();
+ assert!(!owner.snapshot().worker_active);
+ assert!(DictationStream::new(owner, 16000, 1).is_err());
+ assert!(stream.feed(&[0, 0]).is_err());
+ }
+ #[test]
+ fn missing_models_close_capture_once_without_starting_hardware() {
+ for live in [true, false] {
+ let owner = DictationSession::new(live);
+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
+ let signals = Arc::new(StreamSignals::default());
+ assert!(
+ stream
+ .run(
+ crate::transcription::Transcriber::new("missing".into(), "missing".into()),
+ signals.clone()
+ )
+ .is_err()
+ );
+ assert_eq!(signals.ready.load(Ordering::Relaxed), 0);
+ assert_eq!(signals.stopped.load(Ordering::Relaxed), 1);
+ assert!(!owner.snapshot().worker_active);
+ assert!(owner.snapshot().incomplete);
+ }
+ }
+ #[test]
+ fn replacement_matches_the_composer_preview_and_discard_rejects_late_text() {
+ let owner = DictationSession::new(true);
+ let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
+ event(
+ &owner,
+ LiveEvent::Commit {
+ start: 0,
+ end: 16000,
+ text: "partial".into(),
+ },
+ );
+ event(&owner, LiveEvent::Hypothesis("preview".into()));
+ assert!(
+ owner
+ .lock()
+ .apply(
+ LiveEvent::Commit {
+ start: 1,
+ end: 16000,
+ text: "overlap".into()
+ },
+ || true
+ )
+ .is_err()
+ );
+ event(
+ &owner,
+ LiveEvent::ReplaceTranscript("Complete recording".into()),
+ );
+ assert_eq!(owner.snapshot().text, "Complete recording");
+ assert!(owner.snapshot().hypothesis.is_empty());
+ stream.abandon(DictationStopReason::Discard).unwrap();
+ event(&owner, LiveEvent::ReplaceTranscript("late".into()));
+ assert_eq!(owner.snapshot().text, "Complete recording");
+ assert!(matches!(owner.snapshot().phase, DictationPhase::Closed));
+ }
+}
diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.json
index c9add6495a..9280db1850 100644
--- a/rust/checks/lint-exceptions/suppressions.json
+++ b/rust/checks/lint-exceptions/suppressions.json
@@ -27,7 +27,6 @@
"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/platform.rs": ["unsafe_code"],
- "rust/crates/ensu/src/transcription/text.rs": ["clippy::expect_used"],
"rust/crates/exif/src/isobmff.rs": ["clippy::expect_used"],
"rust/crates/image/src/color_management.rs": ["clippy::expect_used"],
"rust/crates/location/src/binary.rs": ["clippy::expect_used"],
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index f2d16c6332..0a66e06b30 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -7,6 +7,7 @@ ente-assets.workspace = true
ente-core.workspace = true
ente-ensu-crypto.workspace = true
ente-image.workspace = true
+ort = { workspace = true, optional = true }
regex.workspace = true
rubato = { version = "5.0.0", optional = true }
rusqlite = { workspace = true, default-features = true, optional = true }
@@ -15,7 +16,7 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "time"] }
-transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }
+transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx"], optional = true }
url.workspace = true
uuid = { workspace = true, features = ["v4", "rng-getrandom"] }
zeroize.workspace = true
diff --git a/rust/crates/ensu/src/lib.rs b/rust/crates/ensu/src/lib.rs
index 9cfb5e7919..1f77467272 100644
--- a/rust/crates/ensu/src/lib.rs
+++ b/rust/crates/ensu/src/lib.rs
@@ -12,3 +12,5 @@ mod platform;
#[cfg(feature = "transcription")]
pub mod transcription;
+#[cfg(feature = "transcription")]
+pub mod voice;
diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rs
index f8bb38792d..0b3d1b9083 100644
--- a/rust/crates/ensu/src/transcription/audio.rs
+++ b/rust/crates/ensu/src/transcription/audio.rs
@@ -3,13 +3,12 @@ use std::time::Duration;
use rubato::audioadapter_buffers::direct::InterleavedSlice;
use rubato::{Fft, FixedSync, Resampler, WindowFunction};
-use transcribe_rs::vad::{SileroVad, SmoothedVad, Vad};
+use transcribe_rs::vad::{SmoothedVad, Vad};
-use crate::transcription::{Result, TranscriptionError};
+use crate::transcription::{Result, SileroV4Vad, TranscriptionError};
const TARGET_SAMPLE_RATE: usize = 16_000;
const RESAMPLER_CHUNK_SIZE: usize = 1024;
-const VAD_THRESHOLD: f32 = 0.5;
const VAD_PREFILL_FRAMES: usize = 3;
const VAD_HANGOVER_FRAMES: usize = 10;
const VAD_ONSET_FRAMES: usize = 2;
@@ -53,7 +52,7 @@ fn extract_speech(
if !vad_model_path.is_file() {
return Err(TranscriptionError::VadNotDownloaded);
}
- let silero = SileroVad::new(vad_model_path, VAD_THRESHOLD)?;
+ let silero = SileroV4Vad::new(vad_model_path)?;
let mut vad = SmoothedVad::new(
Box::new(silero),
VAD_PREFILL_FRAMES,
@@ -61,31 +60,39 @@ fn extract_speech(
VAD_ONSET_FRAMES,
);
+ extract_speech_with_vad(input_sample_rate, samples, &mut vad)
+}
+
+fn extract_speech_with_vad(
+ input_sample_rate: u32,
+ samples: &[f32],
+ vad: &mut dyn Vad,
+) -> Result<Vec<f32>> {
let mut resampler = FrameResampler::new(
input_sample_rate as usize,
TARGET_SAMPLE_RATE,
- Duration::from_millis(30),
+ Duration::from_secs_f64(vad.frame_size() as f64 / TARGET_SAMPLE_RATE as f64),
)?;
let mut speech = Vec::new();
-
- resampler.push(samples, |frame| {
- if let Ok(true) = vad.is_speech(frame) {
- let prefill = vad.drain_prefill();
- if !prefill.is_empty() {
- speech.extend_from_slice(&prefill);
- }
- speech.extend_from_slice(frame);
+ let mut vad_error = None;
+ let mut emit = |frame: &[f32]| {
+ if vad_error.is_some() {
+ return;
}
- });
- resampler.finish(|frame| {
- if let Ok(true) = vad.is_speech(frame) {
- let prefill = vad.drain_prefill();
- if !prefill.is_empty() {
- speech.extend_from_slice(&prefill);
+ match vad.is_speech(frame) {
+ Ok(true) => {
+ speech.extend_from_slice(&vad.drain_prefill());
+ speech.extend_from_slice(frame);
}
- speech.extend_from_slice(frame);
+ Ok(false) => {}
+ Err(error) => vad_error = Some(error),
}
- });
+ };
+ resampler.push(samples, &mut emit);
+ resampler.finish(&mut emit);
+ if let Some(error) = vad_error {
+ return Err(error.into());
+ }
vad.reset();
let min_samples = TARGET_SAMPLE_RATE * 5 / 4;
@@ -200,6 +207,35 @@ mod tests {
use super::*;
+ #[test]
+ fn vad_failure_does_not_return_an_apparently_complete_partial_recording() {
+ struct FailsAfterSpeech {
+ frames: usize,
+ }
+ impl Vad for FailsAfterSpeech {
+ fn frame_size(&self) -> usize {
+ 480
+ }
+ fn is_speech(
+ &mut self,
+ _: &[f32],
+ ) -> std::result::Result<bool, transcribe_rs::TranscribeError> {
+ self.frames += 1;
+ if self.frames == 1 {
+ Ok(true)
+ } else {
+ Err(transcribe_rs::TranscribeError::Inference(
+ "injected VAD failure".into(),
+ ))
+ }
+ }
+ fn reset(&mut self) {}
+ }
+ let mut vad = FailsAfterSpeech { frames: 0 };
+ assert!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad).is_err());
+ assert_eq!(vad.frames, 2);
+ }
+
fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {
let mut resampler =
FrameResampler::new(sample_rate, 16_000, Duration::from_millis(30)).unwrap();
diff --git a/rust/crates/ensu/src/transcription/graph_cache.rs b/rust/crates/ensu/src/transcription/graph_cache.rs
new file mode 100644
index 0000000000..807f739cc6
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/graph_cache.rs
@@ -0,0 +1,651 @@
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Read, Write};
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+use std::time::UNIX_EPOCH;
+
+use ente_ensu_crypto::{Sha256Hasher, sha256};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+pub(super) const GRAPHS: [&str; 3] = [
+ "encoder-model.int8.onnx",
+ "decoder_joint-model.int8.onnx",
+ "nemo128.onnx",
+];
+pub(super) const SOURCES: [&str; 4] = [GRAPHS[0], GRAPHS[1], GRAPHS[2], "vocab.txt"];
+const SCHEMA: u32 = 2;
+const CACHE_DIRECTORY: &str = "ensu-voice-onnx-v1";
+const MAX_BYTES: u64 = 1536 * 1024 * 1024;
+const MAX_MANIFEST: u64 = 64 * 1024;
+
+pub(super) enum LoadPlan<'a> {
+ Source { save_to: Option<&'a Path> },
+ Cached { directory: &'a Path },
+}
+
+pub(super) struct GraphCache {
+ root: PathBuf,
+ device: String,
+ verified: Mutex<Option<Verified>>,
+}
+
+#[derive(PartialEq, Eq)]
+struct Verified {
+ recipe: String,
+ sources: Vec<SourceStamp>,
+ files: Vec<SourceStamp>,
+}
+
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+struct SourceStamp {
+ bytes: u64,
+ modified_ns: u128,
+ #[cfg(unix)]
+ device: u64,
+ #[cfg(unix)]
+ inode: u64,
+ #[cfg(unix)]
+ changed_secs: i64,
+ #[cfg(unix)]
+ changed_ns: i64,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Eq)]
+struct Artifact {
+ bytes: u64,
+ sha256: String,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Manifest {
+ schema: u32,
+ recipe: String,
+ source_stamps: Vec<SourceStamp>,
+ #[cfg(not(unix))]
+ source_sha256: Vec<String>,
+ artifacts: Vec<Artifact>,
+}
+
+struct Staging(PathBuf);
+impl Drop for Staging {
+ fn drop(&mut self) {
+ let _ = remove_owned_directory(&self.0);
+ }
+}
+
+impl GraphCache {
+ pub fn new(cache_directory: PathBuf, device_identity: String) -> Self {
+ Self {
+ root: cache_directory.join(CACHE_DIRECTORY),
+ device: device_identity,
+ verified: Mutex::new(None),
+ }
+ }
+
+ pub fn load<T, E>(
+ &self,
+ model_dir: &Path,
+ runtime_configuration: &str,
+ may_build: bool,
+ mut loader: impl FnMut(LoadPlan<'_>) -> Result<T, E>,
+ ) -> Result<T, E> {
+ let Some((_lock, stamps)) = self.lock_and_stat(model_dir) else {
+ return loader(LoadPlan::Source { save_to: None });
+ };
+ let recipe = digest_bytes(
+ format!(
+ "schema={SCHEMA};upstream=efc66111;ort-api={};os={};arch={};device={};{runtime_configuration}",
+ ort::MINOR_VERSION,
+ std::env::consts::OS,
+ std::env::consts::ARCH,
+ self.device,
+ )
+ .as_bytes(),
+ );
+ let ready = self.root.join("ready");
+ let fingerprint = cache_fingerprint(&ready, &recipe, &stamps).ok();
+ let already_verified = cfg!(unix)
+ && fingerprint.as_ref().is_some_and(|fingerprint| {
+ self.verified
+ .lock()
+ .ok()
+ .is_some_and(|v| v.as_ref() == Some(fingerprint))
+ });
+ if validate(&ready, model_dir, &recipe, &stamps, already_verified).is_ok()
+ && let Ok(model) = loader(LoadPlan::Cached { directory: &ready })
+ && source_stamps(model_dir).ok().as_ref() == Some(&stamps)
+ && cache_fingerprint(&ready, &recipe, &stamps).ok() == fingerprint
+ {
+ if let Ok(mut verified) = self.verified.lock() {
+ *verified = fingerprint;
+ }
+ return Ok(model);
+ }
+ if let Ok(mut verified) = self.verified.lock() {
+ *verified = None;
+ }
+ if !may_build || self.clean_for_build().is_err() {
+ return loader(LoadPlan::Source { save_to: None });
+ }
+ let staging = Staging(self.root.join(format!(".build-{}", Uuid::new_v4())));
+ if fs::create_dir(&staging.0).is_err() {
+ return loader(LoadPlan::Source { save_to: None });
+ }
+ if stamps.last().is_none_or(|stamp| stamp.bytes > MAX_BYTES)
+ || fs::copy(model_dir.join("vocab.txt"), staging.0.join("vocab.txt")).is_err()
+ {
+ return loader(LoadPlan::Source { save_to: None });
+ }
+ #[cfg(not(unix))]
+ let Ok(source_sha256) = source_hashes(model_dir) else {
+ return loader(LoadPlan::Source { save_to: None });
+ };
+ let Ok(model) = loader(LoadPlan::Source {
+ save_to: Some(&staging.0),
+ }) else {
+ drop(staging);
+ return loader(LoadPlan::Source { save_to: None });
+ };
+ let publish = (|| -> io::Result<()> {
+ if source_stamps(model_dir)? != stamps {
+ return Err(invalid_cache());
+ }
+ let artifacts = artifacts(&staging.0)?;
+ let manifest = Manifest {
+ schema: SCHEMA,
+ recipe: recipe.clone(),
+ source_stamps: stamps.clone(),
+ #[cfg(not(unix))]
+ source_sha256,
+ artifacts,
+ };
+ let mut file = OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(staging.0.join("manifest.json"))?;
+ file.write_all(&serde_json::to_vec(&manifest)?)?;
+ file.sync_all()?;
+ fs::rename(&staging.0, &ready)?;
+ Ok(())
+ })();
+ if publish.is_ok()
+ && let Ok(mut verified) = self.verified.lock()
+ {
+ *verified = cache_fingerprint(&ready, &recipe, &stamps).ok();
+ }
+ Ok(model)
+ }
+
+ fn lock_and_stat(&self, model_dir: &Path) -> Option<(File, Vec<SourceStamp>)> {
+ fs::create_dir_all(&self.root).ok()?;
+ if !fs::symlink_metadata(&self.root).ok()?.is_dir() {
+ return None;
+ }
+ let lock = OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .read(true)
+ .write(true)
+ .open(self.root.join(".lock"))
+ .ok()?;
+ lock.try_lock().ok()?;
+ Some((lock, source_stamps(model_dir).ok()?))
+ }
+
+ fn clean_for_build(&self) -> io::Result<()> {
+ remove_owned_directory(&self.root.join("ready"))?;
+ for entry in fs::read_dir(&self.root)? {
+ let entry = entry?;
+ let name = entry.file_name();
+ if name
+ .to_str()
+ .and_then(|s| s.strip_prefix(".build-"))
+ .and_then(|s| Uuid::parse_str(s).ok())
+ .is_some()
+ {
+ remove_owned_directory(&entry.path())?;
+ }
+ }
+ Ok(())
+ }
+}
+
+fn source_stamps(model_dir: &Path) -> io::Result<Vec<SourceStamp>> {
+ stamps_for(model_dir, &SOURCES)
+}
+
+fn cache_fingerprint(ready: &Path, recipe: &str, sources: &[SourceStamp]) -> io::Result<Verified> {
+ Ok(Verified {
+ recipe: recipe.into(),
+ sources: sources.to_vec(),
+ files: stamps_for(
+ ready,
+ &[
+ GRAPHS[0],
+ GRAPHS[1],
+ GRAPHS[2],
+ "vocab.txt",
+ "manifest.json",
+ ],
+ )?,
+ })
+}
+
+fn stamps_for(directory: &Path, names: &[&str]) -> io::Result<Vec<SourceStamp>> {
+ names
+ .iter()
+ .map(|name| {
+ let metadata = fs::symlink_metadata(directory.join(name))?;
+ if !metadata.is_file() || metadata.len() == 0 {
+ return Err(invalid_cache());
+ }
+ let modified_ns = metadata
+ .modified()?
+ .duration_since(UNIX_EPOCH)
+ .map_err(|_| invalid_cache())?
+ .as_nanos();
+ #[cfg(unix)]
+ use std::os::unix::fs::MetadataExt;
+ Ok(SourceStamp {
+ bytes: metadata.len(),
+ modified_ns,
+ #[cfg(unix)]
+ device: metadata.dev(),
+ #[cfg(unix)]
+ inode: metadata.ino(),
+ #[cfg(unix)]
+ changed_secs: metadata.ctime(),
+ #[cfg(unix)]
+ changed_ns: metadata.ctime_nsec(),
+ })
+ })
+ .collect()
+}
+
+#[cfg(not(unix))]
+fn source_hashes(model_dir: &Path) -> io::Result<Vec<String>> {
+ SOURCES
+ .iter()
+ .map(|name| digest_file(&model_dir.join(name)).map(|a| a.sha256))
+ .collect()
+}
+
+fn validate(
+ ready: &Path,
+ model_dir: &Path,
+ recipe: &str,
+ stamps: &[SourceStamp],
+ already_verified: bool,
+) -> io::Result<()> {
+ if !fs::symlink_metadata(ready)?.is_dir() {
+ return Err(invalid_cache());
+ }
+ let path = ready.join("manifest.json");
+ let metadata = fs::symlink_metadata(&path)?;
+ if !metadata.is_file() || metadata.len() > MAX_MANIFEST {
+ return Err(invalid_cache());
+ }
+ let mut bytes = Vec::new();
+ File::open(path)?
+ .take(MAX_MANIFEST + 1)
+ .read_to_end(&mut bytes)?;
+ if bytes.len() as u64 > MAX_MANIFEST {
+ return Err(invalid_cache());
+ }
+ let manifest: Manifest = serde_json::from_slice(&bytes)?;
+ if manifest.schema != SCHEMA
+ || manifest.recipe != recipe
+ || manifest.source_stamps != stamps
+ || manifest.artifacts.len() != SOURCES.len()
+ {
+ return Err(invalid_cache());
+ }
+ #[cfg(not(unix))]
+ if source_hashes(model_dir)? != manifest.source_sha256 {
+ return Err(invalid_cache());
+ }
+ if already_verified {
+ validate_directory_entries(ready)?;
+ let bytes: u64 = manifest.artifacts.iter().map(|g| g.bytes).sum();
+ if bytes > MAX_BYTES {
+ return Err(invalid_cache());
+ }
+ return Ok(());
+ }
+ if artifacts(ready)? != manifest.artifacts || source_stamps(model_dir)? != stamps {
+ return Err(invalid_cache());
+ }
+ Ok(())
+}
+
+fn artifacts(directory: &Path) -> io::Result<Vec<Artifact>> {
+ let mut total = 0;
+ let mut artifacts = Vec::new();
+ for name in SOURCES {
+ let artifact = digest_file(&directory.join(name))?;
+ total += artifact.bytes;
+ if total > MAX_BYTES {
+ return Err(invalid_cache());
+ }
+ artifacts.push(artifact);
+ }
+ validate_directory_entries(directory)?;
+ Ok(artifacts)
+}
+
+fn validate_directory_entries(directory: &Path) -> io::Result<()> {
+ for entry in fs::read_dir(directory)? {
+ let name = entry?.file_name();
+ if !SOURCES.iter().any(|expected| name == *expected) && name != "manifest.json" {
+ return Err(invalid_cache());
+ }
+ }
+ Ok(())
+}
+
+fn digest_file(path: &Path) -> io::Result<Artifact> {
+ let metadata = fs::symlink_metadata(path)?;
+ if !metadata.is_file() || metadata.len() == 0 || metadata.len() > MAX_BYTES {
+ return Err(invalid_cache());
+ }
+ let mut file = File::open(path)?;
+ let mut hasher = Sha256Hasher::new();
+ let mut buffer = vec![0_u8; 64 * 1024];
+ let mut bytes = 0;
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ bytes += read as u64;
+ if bytes > MAX_BYTES {
+ return Err(invalid_cache());
+ }
+ hasher.update(&buffer[..read]);
+ }
+ if bytes != metadata.len() {
+ return Err(invalid_cache());
+ }
+ Ok(Artifact {
+ bytes,
+ sha256: hex(&hasher.finalize()),
+ })
+}
+
+fn digest_bytes(bytes: &[u8]) -> String {
+ hex(&sha256(bytes))
+}
+
+fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+fn invalid_cache() -> io::Error {
+ io::Error::new(io::ErrorKind::InvalidData, "Invalid voice graph cache")
+}
+fn remove_owned_directory(path: &Path) -> io::Result<()> {
+ match fs::symlink_metadata(path) {
+ Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path),
+ Ok(_) => fs::remove_file(path),
+ Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
+ Err(error) => Err(error),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::cell::RefCell;
+
+ struct Fixture {
+ temp: tempfile::TempDir,
+ source: PathBuf,
+ cache: GraphCache,
+ loads: RefCell<Vec<&'static str>>,
+ }
+
+ impl Fixture {
+ fn new() -> Self {
+ let temp = tempfile::tempdir().unwrap();
+ let source = temp.path().join("source");
+ fs::create_dir(&source).unwrap();
+ for name in SOURCES {
+ fs::write(source.join(name), name.as_bytes()).unwrap();
+ }
+ let cache = GraphCache::new(temp.path().join("cache"), "phone-a".into());
+ Self {
+ temp,
+ source,
+ cache,
+ loads: RefCell::new(Vec::new()),
+ }
+ }
+
+ fn load(&self, recipe: &str) -> String {
+ self.cache
+ .load(&self.source, recipe, true, |plan| self.loader(plan))
+ .unwrap()
+ }
+
+ fn loader(&self, plan: LoadPlan<'_>) -> io::Result<String> {
+ let (directory, save_to) = match plan {
+ LoadPlan::Source { save_to } => {
+ self.loads.borrow_mut().push(if save_to.is_some() {
+ "build"
+ } else {
+ "source"
+ });
+ (self.source.as_path(), save_to)
+ }
+ LoadPlan::Cached { directory } => {
+ self.loads.borrow_mut().push("hit");
+ (directory, None)
+ }
+ };
+ let mut result = String::new();
+ for name in GRAPHS {
+ let bytes = fs::read(directory.join(name))?;
+ if let Some(save_to) = save_to {
+ fs::write(save_to.join(name), &bytes)?;
+ }
+ result.push_str(&digest_bytes(&bytes));
+ }
+ Ok(result)
+ }
+
+ fn ready(&self) -> PathBuf {
+ self.cache.root.join("ready")
+ }
+ }
+
+ #[test]
+ fn deferred_load_reuses_but_never_builds_graphs() {
+ let f = Fixture::new();
+ let expected = f
+ .cache
+ .load(&f.source, "runtime-a", false, |plan| f.loader(plan))
+ .unwrap();
+ assert!(!f.ready().exists());
+ assert_eq!(f.load("runtime-a"), expected);
+ assert_eq!(f.load("runtime-a"), expected);
+ assert!(f.cache.verified.lock().unwrap().is_some());
+ let restarted = GraphCache::new(f.temp.path().join("cache"), "phone-a".into());
+ assert_eq!(
+ restarted
+ .load(&f.source, "runtime-a", false, |plan| f.loader(plan))
+ .unwrap(),
+ expected
+ );
+ assert!(restarted.verified.lock().unwrap().is_some());
+ assert_eq!(f.loads.take(), ["source", "build", "hit", "hit"]);
+ }
+
+ #[test]
+ fn changed_graph_source_runtime_device_and_manifest_rebuild() {
+ let mut f = Fixture::new();
+ let first = f.load("runtime-a;cpu");
+ f.loads.take();
+ for name in [GRAPHS[0], "vocab.txt"] {
+ let path = f.ready().join(name);
+ let mut bytes = fs::read(&path).unwrap();
+ bytes[0] ^= 1;
+ fs::write(&path, bytes).unwrap();
+ assert_eq!(f.load("runtime-a;cpu"), first);
+ assert_eq!(f.loads.take(), ["build"]);
+ assert_eq!(
+ fs::read(path).unwrap(),
+ fs::read(f.source.join(name)).unwrap()
+ );
+ }
+
+ let source = f.source.join(GRAPHS[0]);
+ let mtime = fs::metadata(&source).unwrap().modified().unwrap();
+ let mut bytes = fs::read(&source).unwrap();
+ bytes[0] ^= 1;
+ fs::write(&source, bytes).unwrap();
+ File::options()
+ .write(true)
+ .open(source)
+ .unwrap()
+ .set_times(fs::FileTimes::new().set_modified(mtime))
+ .unwrap();
+ let updated = f.load("runtime-a;cpu");
+ assert_ne!(updated, first);
+ assert_eq!(f.loads.take(), ["build"]);
+ for recipe in ["runtime-b;cpu", "runtime-b;xnnpack"] {
+ assert_eq!(f.load(recipe), updated);
+ assert_eq!(f.loads.take(), ["build"]);
+ }
+ f.cache.device = "phone-b".into();
+ assert_eq!(f.load("runtime-b;xnnpack"), updated);
+ assert_eq!(f.loads.take(), ["build"]);
+ fs::write(f.ready().join("manifest.json"), b"{partial").unwrap();
+ let abandoned = f.cache.root.join(format!(".build-{}", Uuid::new_v4()));
+ fs::create_dir(&abandoned).unwrap();
+ fs::write(abandoned.join("partial"), b"incomplete").unwrap();
+ fs::write(f.cache.root.join("unrelated"), b"keep").unwrap();
+ assert_eq!(f.load("runtime-b;xnnpack"), updated);
+ assert_eq!(f.loads.take(), ["build"]);
+ assert!(!abandoned.exists());
+ assert_eq!(fs::read(f.cache.root.join("unrelated")).unwrap(), b"keep");
+ assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 3);
+ }
+
+ #[test]
+ fn missing_original_never_loads_a_surviving_cache() {
+ let f = Fixture::new();
+ f.load("runtime-a");
+ fs::remove_file(f.source.join(GRAPHS[0])).unwrap();
+ assert!(
+ f.cache
+ .load(&f.source, "runtime-a", true, |plan| f.loader(plan))
+ .is_err()
+ );
+ assert_eq!(f.loads.take(), ["build", "source"]);
+ }
+
+ #[test]
+ fn serializer_failure_retries_original_and_cleans_staging() {
+ let f = Fixture::new();
+ let result = f.cache.load(&f.source, "runtime-a", true, |plan| {
+ if matches!(plan, LoadPlan::Source { save_to: Some(_) }) {
+ f.loads.borrow_mut().push("failed-save");
+ Err(io::Error::new(
+ io::ErrorKind::StorageFull,
+ "injected full disk",
+ ))
+ } else {
+ f.loader(plan)
+ }
+ });
+ assert!(result.is_ok());
+ assert_eq!(f.loads.take(), ["failed-save", "source"]);
+ assert!(!f.ready().exists());
+ assert_eq!(fs::read_dir(&f.cache.root).unwrap().count(), 1);
+ }
+
+ #[test]
+ fn rejected_hash_valid_graph_rebuilds_from_source() {
+ let f = Fixture::new();
+ let expected = f.load("runtime-a");
+ let actual = f
+ .cache
+ .load(&f.source, "runtime-a", true, |plan| {
+ if matches!(plan, LoadPlan::Cached { .. }) {
+ Err(invalid_cache())
+ } else {
+ f.loader(plan)
+ }
+ })
+ .unwrap();
+ assert_eq!(actual, expected);
+ assert_eq!(f.loads.take(), ["build", "build"]);
+ }
+
+ #[test]
+ fn busy_or_unusable_cache_never_blocks_source_loading() {
+ let f = Fixture::new();
+ let expected = f.load("runtime-a");
+ let lock = File::options()
+ .read(true)
+ .write(true)
+ .open(f.cache.root.join(".lock"))
+ .unwrap();
+ lock.try_lock().unwrap();
+ assert_eq!(f.load("runtime-a"), expected);
+ drop(lock);
+ let file = f.temp.path().join("not-a-directory");
+ fs::write(&file, b"existing").unwrap();
+ let cache = GraphCache::new(file, "phone-a".into());
+ assert_eq!(
+ cache
+ .load(&f.source, "runtime-a", true, |plan| f.loader(plan))
+ .unwrap(),
+ expected
+ );
+ assert_eq!(f.loads.take(), ["build", "source", "source"]);
+ }
+
+ #[test]
+ fn oversized_or_external_output_is_not_published() {
+ let f = Fixture::new();
+ for oversize in [true, false] {
+ let result = f.cache.load(&f.source, "runtime-a", true, |plan| {
+ let save_to = match &plan {
+ LoadPlan::Source { save_to } => *save_to,
+ _ => None,
+ };
+ let result = f.loader(plan)?;
+ if let Some(path) = save_to {
+ if oversize {
+ File::options()
+ .write(true)
+ .open(path.join(GRAPHS[0]))?
+ .set_len(MAX_BYTES + 1)?;
+ } else {
+ fs::write(path.join("unverified.weights"), b"external")?;
+ }
+ }
+ Ok::<_, io::Error>(result)
+ });
+ assert!(result.is_ok());
+ assert!(!f.ready().exists());
+ }
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn ready_symlink_cannot_redirect_loading_or_cleanup() {
+ let f = Fixture::new();
+ let expected = f.load("runtime-a");
+ let outside = f.temp.path().join("outside");
+ fs::create_dir(&outside).unwrap();
+ fs::write(outside.join("preserve"), b"original").unwrap();
+ fs::remove_dir_all(f.ready()).unwrap();
+ std::os::unix::fs::symlink(&outside, f.ready()).unwrap();
+ assert_eq!(f.load("runtime-a"), expected);
+ assert_eq!(f.loads.take(), ["build", "build"]);
+ assert_eq!(fs::read(outside.join("preserve")).unwrap(), b"original");
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rs
new file mode 100644
index 0000000000..33960e7154
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/live.rs
@@ -0,0 +1,642 @@
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{self, TryRecvError};
+use std::time::Duration;
+
+use transcribe_rs::vad::Vad;
+
+use super::{StreamNormalizer, Transcriber};
+use crate::voice::{
+ AlignedWord, CaptureFormat, CaptureQueue, Result, RollingRecognizer, VoiceError,
+};
+
+pub enum LiveEvent {
+ Ready,
+ Audio { start: u64, end: u64, voiced: bool },
+ Hypothesis(String),
+ Commit { start: u64, end: u64, text: String },
+ Deferred,
+ ReplaceTranscript(String),
+ Coverage(u64),
+ Idle,
+ CaptureClosed,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum LiveControl {
+ Continue,
+ Finish,
+ Cancel,
+}
+
+enum DecoderEvent {
+ Prepared(Result<()>),
+ Window(crate::voice::DecodeWindow, Result<Vec<AlignedWord>>),
+}
+
+pub struct LiveTranscription {
+ pub(super) queue: CaptureQueue,
+ started: AtomicBool,
+}
+
+pub(super) struct CaptureGuard<'a>(&'a CaptureQueue);
+impl Drop for CaptureGuard<'_> {
+ fn drop(&mut self) {
+ self.0.clear();
+ }
+}
+
+impl LiveTranscription {
+ pub fn new(format: CaptureFormat) -> Result<Self> {
+ Ok(Self {
+ queue: CaptureQueue::new(format)?,
+ started: AtomicBool::new(false),
+ })
+ }
+
+ pub fn feed(&self, pcm: &[u8]) -> Result<()> {
+ self.queue.push(pcm)
+ }
+ pub fn finish(&self) {
+ self.queue.finish();
+ }
+ pub fn finish_if_empty(&self) -> bool {
+ self.queue.finish_if_empty()
+ }
+ pub fn cancel(&self) {
+ self.queue.cancel();
+ }
+ pub fn submitted_samples(&self) -> u64 {
+ self.queue.submitted_frames() * 16000 / u64::from(self.queue.format().sample_rate)
+ }
+
+ pub(super) fn start(&self) -> Result<CaptureGuard<'_>> {
+ if self.started.swap(true, Ordering::AcqRel) {
+ return Err(VoiceError::InvalidState);
+ }
+ let guard = CaptureGuard(&self.queue);
+ self.check()?;
+ Ok(guard)
+ }
+
+ pub(super) fn check(&self) -> Result<()> {
+ if self.queue.is_cancelled() {
+ Err(VoiceError::Cancelled)
+ } else if self.queue.failed() {
+ Err(VoiceError::BacklogFull)
+ } else {
+ Ok(())
+ }
+ }
+
+ pub(super) fn emit(
+ &self,
+ event: LiveEvent,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ let idle = matches!(event, LiveEvent::Idle);
+ match on_event(event)? {
+ LiveControl::Cancel => {
+ self.cancel();
+ return Err(VoiceError::Cancelled);
+ }
+ LiveControl::Finish if idle => {
+ self.queue.finish_if_empty();
+ }
+ LiveControl::Finish => self.queue.finish(),
+ LiveControl::Continue => {}
+ }
+ Ok(())
+ }
+
+ pub fn run(
+ &self,
+ transcriber: &Transcriber,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ self.run_with(
+ || transcriber.live_vad(),
+ || {
+ transcriber
+ .load_model()
+ .map_err(|_| VoiceError::ProcessingFailed)
+ },
+ |request| transcriber.transcribe_window(&request.samples),
+ |pcm| {
+ transcriber
+ .transcribe(16_000, pcm)
+ .map_err(|_| VoiceError::ProcessingFailed)
+ },
+ on_event,
+ )
+ }
+
+ fn run_with(
+ &self,
+ make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,
+ prepare: impl FnOnce() -> Result<()> + Send,
+ mut decode: impl FnMut(&crate::voice::DecodeWindow) -> Result<Vec<AlignedWord>> + Send,
+ decode_recorded: impl FnOnce(&[u8]) -> Result<String>,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ let _capture = self.start()?;
+ let mut vad = make_vad()?;
+ if vad.frame_size() != super::VAD_FRAME_SAMPLES {
+ return Err(VoiceError::InvalidAudio);
+ }
+ let mut normalizer = StreamNormalizer::new(self.queue.format())?;
+ let mut rolling = RollingRecognizer::default();
+ let mut pcm = super::recorded::RecordedPcm::default();
+ let mut accepted = 0_u64;
+ let mut deferred = false;
+ self.emit(LiveEvent::Ready, on_event)?;
+ std::thread::scope(|scope| {
+ let (requests, request_rx) = mpsc::sync_channel::<crate::voice::DecodeWindow>(1);
+ let (results, result_rx) = mpsc::sync_channel(1);
+ let decoder = scope.spawn(move || {
+ let prepared = if self.queue.is_cancelled() {
+ Err(VoiceError::Cancelled)
+ } else {
+ prepare()
+ };
+ let failed = prepared.is_err();
+ if results.send(DecoderEvent::Prepared(prepared)).is_err() || failed {
+ return;
+ }
+ while let Ok(request) = request_rx.recv() {
+ let output = if self.queue.is_cancelled() {
+ Err(VoiceError::Cancelled)
+ } else {
+ decode(&request)
+ };
+ if results.send(DecoderEvent::Window(request, output)).is_err() {
+ break;
+ }
+ }
+ });
+ let mut finished = false;
+ let mut prepared = false;
+ let mut notified_stop = false;
+ let processing = (|| -> Result<()> {
+ loop {
+ self.check()?;
+ let chunk = self.queue.take();
+ let had_chunk = chunk.is_some();
+ let mut emit = |frame: &[f32], real: usize| -> Result<()> {
+ let voiced = vad
+ .is_speech(frame)
+ .map_err(|_| VoiceError::ProcessingFailed)?;
+ let start = accepted;
+ accepted += real as u64;
+ self.emit(
+ LiveEvent::Audio {
+ start,
+ end: accepted,
+ voiced,
+ },
+ on_event,
+ )?;
+ pcm.push(&frame[..real])?;
+ if !deferred {
+ match rolling.feed(&frame[..real], voiced) {
+ Ok(()) => {}
+ Err(VoiceError::BacklogFull) => {
+ deferred = true;
+ self.emit(LiveEvent::Deferred, on_event)?;
+ }
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(())
+ };
+ if let Some(chunk) = chunk {
+ normalizer.push(&chunk, &mut emit)?;
+ }
+ if !finished && self.queue.is_drained() {
+ normalizer.finish(&mut emit)?;
+ if !deferred {
+ rolling.finish();
+ }
+ finished = true;
+ }
+ if self.queue.is_closed() && !notified_stop {
+ self.emit(LiveEvent::CaptureClosed, on_event)?;
+ notified_stop = true;
+ }
+ match result_rx.try_recv() {
+ Ok(DecoderEvent::Prepared(result)) => {
+ result?;
+ prepared = true;
+ }
+ Ok(DecoderEvent::Window(request, words)) => {
+ if self.queue.is_cancelled() {
+ return Err(VoiceError::Cancelled);
+ }
+ if !deferred {
+ match words.and_then(|words| rolling.complete(&request, words)) {
+ Ok(update) => {
+ if let Some((start, end, text)) = update.commit {
+ self.emit(
+ LiveEvent::Commit { start, end, text },
+ on_event,
+ )?;
+ }
+ self.emit(
+ LiveEvent::Hypothesis(update.hypothesis),
+ on_event,
+ )?;
+ self.emit(
+ LiveEvent::Coverage(update.covered_through),
+ on_event,
+ )?;
+ }
+ Err(
+ VoiceError::UnstableTranscript
+ | VoiceError::IncompleteAudio,
+ ) => {
+ deferred = true;
+ self.emit(LiveEvent::Deferred, on_event)?;
+ }
+ Err(error) => return Err(error),
+ }
+ }
+ }
+ Err(TryRecvError::Empty) => {}
+ Err(TryRecvError::Disconnected) => {
+ return Err(VoiceError::ProcessingFailed);
+ }
+ }
+ if prepared && finished && (deferred || rolling.is_settled()) {
+ if !deferred {
+ self.emit(LiveEvent::Coverage(accepted), on_event)?;
+ }
+ return Ok(());
+ }
+ if prepared
+ && !deferred
+ && let Some(request) = rolling.next_window()?
+ {
+ requests
+ .try_send(request)
+ .map_err(|_| VoiceError::ProcessingFailed)?;
+ }
+ if !had_chunk && !finished {
+ self.emit(LiveEvent::Idle, on_event)?;
+ }
+ if !had_chunk {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ }
+ })();
+ self.queue.finish();
+ if !notified_stop {
+ let _ = on_event(LiveEvent::CaptureClosed);
+ }
+ drop(requests);
+ drop(result_rx);
+ let exited = decoder.join().map_err(|_| VoiceError::ProcessingFailed);
+ exited.and(processing)?;
+ if deferred {
+ drop(vad);
+ drop(normalizer);
+ self.check()?;
+ let text = decode_recorded(pcm.bytes())?;
+ self.check()?;
+ if text.trim().is_empty() {
+ return Err(VoiceError::IncompleteAudio);
+ }
+ self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;
+ self.emit(LiveEvent::Coverage(accepted), on_event)?;
+ }
+ Ok(())
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use transcribe_rs::vad::EnergyVad;
+
+ fn stream() -> LiveTranscription {
+ LiveTranscription::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap()
+ }
+
+ fn vad() -> Result<Box<dyn Vad>> {
+ Ok(Box::new(EnergyVad::new(
+ super::super::VAD_FRAME_SAMPLES,
+ 0.01,
+ )))
+ }
+
+ #[test]
+ fn capture_and_vad_preserve_opening_audio_while_model_loads() {
+ let stream = stream();
+ let (release, loading) = mpsc::sync_channel(1);
+ let loaded = &AtomicBool::new(false);
+ let mut coverage = 0;
+ let mut recognized = false;
+ let prepared = AtomicBool::new(false);
+ let mut closed = 0;
+ stream
+ .run_with(
+ || {
+ prepared.store(true, Ordering::Release);
+ vad()
+ },
+ move || {
+ loading.recv_timeout(Duration::from_secs(3)).unwrap();
+ loaded.store(true, Ordering::Release);
+ Ok(())
+ },
+ |request| {
+ assert!(loaded.load(Ordering::Acquire));
+ assert_eq!(request.samples, vec![0.25; 10241]);
+ Ok(vec![AlignedWord {
+ text: "Opening words".into(),
+ start: 0,
+ end: 10241,
+ }])
+ },
+ |_| panic!("This capture must stay live"),
+ &mut |event| {
+ match event {
+ LiveEvent::Ready => {
+ assert!(prepared.load(Ordering::Acquire));
+ assert!(!loaded.load(Ordering::Acquire));
+ for samples in [2560, 2560, 2560, 2561] {
+ stream.feed(&[0, 32].repeat(samples))?;
+ }
+ }
+ LiveEvent::Audio { end, .. } => {
+ if end <= 10240 {
+ assert!(!loaded.load(Ordering::Acquire));
+ }
+ if end == 10240 {
+ stream.finish();
+ release.send(()).unwrap();
+ }
+ }
+ LiveEvent::CaptureClosed => closed += 1,
+ LiveEvent::Commit { text, .. } => recognized |= text == "Opening words",
+ LiveEvent::Coverage(end) => coverage = end,
+ _ => {}
+ }
+ Ok(LiveControl::Continue)
+ },
+ )
+ .unwrap();
+ assert!(recognized);
+ assert_eq!(coverage, 10241);
+ assert_eq!(stream.submitted_samples(), 10241);
+ assert_eq!(closed, 1);
+ }
+
+ #[test]
+ fn slow_initialization_uses_existing_recorded_recovery_without_losing_audio() {
+ let stream = stream();
+ let (release, loading) = mpsc::sync_channel(1);
+ let mut accepted = 0;
+ let mut deferred = false;
+ let mut replacement = None;
+ let mut coverage = 0;
+ stream
+ .run_with(
+ vad,
+ move || {
+ loading.recv_timeout(Duration::from_secs(3)).unwrap();
+ Ok(())
+ },
+ |_| panic!("Initialization exceeded the existing live backlog limit"),
+ |pcm| {
+ assert_eq!(pcm, [0, 32].repeat(12 * 16000));
+ Ok("All opening speech".into())
+ },
+ &mut |event| {
+ match event {
+ LiveEvent::Audio { end, .. } => accepted = end,
+ LiveEvent::Idle => {
+ if accepted < 12 * 16000 {
+ for _ in 0..5 {
+ stream.feed(&[0, 32].repeat(3200))?;
+ }
+ } else {
+ assert!(deferred);
+ stream.finish();
+ release.send(()).unwrap();
+ }
+ }
+ LiveEvent::Deferred => deferred = true,
+ LiveEvent::ReplaceTranscript(text) => replacement = Some(text),
+ LiveEvent::Coverage(end) => coverage = end,
+ _ => {}
+ }
+ Ok(LiveControl::Continue)
+ },
+ )
+ .unwrap();
+ assert_eq!(replacement.as_deref(), Some("All opening speech"));
+ assert_eq!(coverage, 12 * 16000);
+ }
+
+ #[test]
+ fn stop_and_cancel_during_loading_close_capture_before_joining_the_worker() {
+ for cancel in [false, true] {
+ let stream = stream();
+ let (release, loading) = mpsc::sync_channel(1);
+ let exited = &AtomicBool::new(false);
+ let started = &AtomicBool::new(false);
+ let mut closed = 0;
+ let result = stream.run_with(
+ vad,
+ move || {
+ started.store(true, Ordering::Release);
+ loading.recv_timeout(Duration::from_secs(3)).unwrap();
+ exited.store(true, Ordering::Release);
+ Ok(())
+ },
+ |_| panic!("No audio to decode"),
+ |_| panic!("No recorded fallback"),
+ &mut |event| {
+ match event {
+ LiveEvent::Ready if !cancel => stream.finish(),
+ LiveEvent::Idle if cancel && started.load(Ordering::Acquire) => {
+ stream.cancel()
+ }
+ LiveEvent::CaptureClosed => {
+ assert!(!exited.load(Ordering::Acquire));
+ closed += 1;
+ release.send(()).unwrap();
+ }
+ LiveEvent::Ready | LiveEvent::Idle | LiveEvent::Coverage(0) => {}
+ _ => panic!("Initialization emitted recognition"),
+ }
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(
+ result,
+ if cancel {
+ Err(VoiceError::Cancelled)
+ } else {
+ Ok(())
+ }
+ );
+ assert_eq!(closed, 1);
+ assert!(exited.load(Ordering::Acquire));
+ }
+ }
+
+ #[test]
+ fn failed_model_initialization_closes_capture_without_recognition() {
+ let stream = stream();
+ let mut closed = false;
+ let result = stream.run_with(
+ vad,
+ || Err(VoiceError::ProcessingFailed),
+ |_| panic!("Failed model must not decode"),
+ |_| panic!("Failed model must not transcribe"),
+ &mut |event| {
+ match event {
+ LiveEvent::Ready | LiveEvent::Idle => {}
+ LiveEvent::CaptureClosed => closed = true,
+ _ => panic!("Failed initialization emitted recognition"),
+ }
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(result, Err(VoiceError::ProcessingFailed));
+ assert!(closed);
+ }
+
+ #[test]
+ fn failed_live_decode_continues_capture_and_replays_every_sample_once() {
+ for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {
+ let stream = stream();
+ for _ in 0..3 {
+ stream.feed(&[0, 32].repeat(3200)).unwrap();
+ }
+ let mut deferred = false;
+ let mut closed = false;
+ let mut final_text = Vec::new();
+ let mut coverage = 0;
+ stream
+ .run_with(
+ vad,
+ || Ok(()),
+ |_| Err(failure.clone()),
+ |pcm| {
+ assert_eq!(pcm, [0, 32].repeat(9600 + 1001));
+ Ok("Complete recording".into())
+ },
+ &mut |event| {
+ match event {
+ LiveEvent::Deferred => {
+ assert!(!deferred);
+ assert!(!closed);
+ deferred = true;
+ stream.feed(&[0, 32].repeat(1001))?;
+ stream.finish();
+ }
+ LiveEvent::CaptureClosed => {
+ assert!(deferred);
+ closed = true;
+ }
+ LiveEvent::ReplaceTranscript(text) => {
+ assert!(closed);
+ final_text.push(text);
+ }
+ LiveEvent::Coverage(end) => coverage = end,
+ _ => {}
+ }
+ Ok(LiveControl::Continue)
+ },
+ )
+ .unwrap();
+ assert_eq!(final_text, ["Complete recording"]);
+ assert_eq!(coverage, 10601);
+ }
+ }
+
+ #[test]
+ fn incomplete_or_cancelled_recovery_cannot_publish_text_or_coverage() {
+ for cancelled in [false, true] {
+ let stream = stream();
+ stream.feed(&[0, 32].repeat(1001)).unwrap();
+ stream.finish();
+ let result = stream.run_with(
+ vad,
+ || Ok(()),
+ |_| Err(VoiceError::UnstableTranscript),
+ |_| {
+ if cancelled {
+ stream.cancel();
+ Ok("late result".into())
+ } else {
+ Ok(String::new())
+ }
+ },
+ &mut |event| {
+ assert!(!matches!(
+ event,
+ LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)
+ ));
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(
+ result,
+ Err(if cancelled {
+ VoiceError::Cancelled
+ } else {
+ VoiceError::IncompleteAudio
+ })
+ );
+ }
+ }
+
+ #[test]
+ fn vad_keeps_running_during_decode_and_cancel_joins_native_work() {
+ let stream = stream();
+ for _ in 0..3 {
+ stream.feed(&[1, 32].repeat(3200)).unwrap();
+ }
+ let decoded = &AtomicBool::new(false);
+ let exited = &AtomicBool::new(false);
+ let (release, decoding) = mpsc::sync_channel(1);
+ let mut observed_during_decode = false;
+ let mut fed = false;
+ let result = stream.run_with(
+ vad,
+ || Ok(()),
+ move |_| {
+ decoded.store(true, Ordering::Release);
+ decoding.recv_timeout(Duration::from_secs(3)).unwrap();
+ exited.store(true, Ordering::Release);
+ Ok(Vec::new())
+ },
+ |_| panic!("Cancelled decode must not transcribe"),
+ &mut |event| {
+ match event {
+ LiveEvent::Idle if !fed && decoded.load(Ordering::Acquire) => {
+ stream.feed(&[1, 32].repeat(3200))?;
+ fed = true;
+ }
+ LiveEvent::Audio { end, .. } if end > 9600 => {
+ assert!(!exited.load(Ordering::Acquire));
+ observed_during_decode = true;
+ return Ok(LiveControl::Cancel);
+ }
+ LiveEvent::CaptureClosed => release.send(()).unwrap(),
+ _ => {}
+ }
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(result, Err(VoiceError::Cancelled));
+ assert!(observed_during_decode && exited.load(Ordering::Acquire));
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rs
index 1d11862fc8..651de4da09 100644
--- a/rust/crates/ensu/src/transcription/mod.rs
+++ b/rust/crates/ensu/src/transcription/mod.rs
@@ -1,8 +1,17 @@
mod audio;
-mod text;
+mod graph_cache;
+mod live;
+mod normalize;
+mod recorded;
+mod session;
mod transcriber;
+mod vad;
+pub use live::{LiveControl, LiveEvent, LiveTranscription};
+pub(crate) use normalize::{StreamNormalizer, VAD_FRAME_SAMPLES};
+pub use session::DictationState;
pub use transcriber::Transcriber;
+pub(crate) use vad::SileroV4Vad;
#[derive(Debug, thiserror::Error)]
pub enum TranscriptionError {
@@ -36,6 +45,12 @@ impl From<transcribe_rs::TranscribeError> for TranscriptionError {
}
}
+impl<R> From<ort::Error<R>> for TranscriptionError {
+ fn from(value: ort::Error<R>) -> Self {
+ Self::Transcribe(value.to_string())
+ }
+}
+
impl From<rubato::ResamplerConstructionError> for TranscriptionError {
fn from(value: rubato::ResamplerConstructionError) -> Self {
Self::InvalidAudio(value.to_string())
diff --git a/rust/crates/ensu/src/transcription/normalize.rs b/rust/crates/ensu/src/transcription/normalize.rs
new file mode 100644
index 0000000000..66e50b9d98
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/normalize.rs
@@ -0,0 +1,205 @@
+use rubato::audioadapter_buffers::direct::InterleavedSlice;
+use rubato::{Fft, FixedSync, Resampler, WindowFunction};
+
+use crate::voice::{CaptureFormat, Result, VoiceError};
+
+pub const VAD_FRAME_SAMPLES: usize = super::vad::SILERO_V4_FRAME_SAMPLES;
+
+pub(crate) struct StreamNormalizer {
+ format: CaptureFormat,
+ resampler: Option<Fft<f32>>,
+ input: Vec<f32>,
+ frame: Vec<f32>,
+ delay: usize,
+ input_frames: u64,
+ emitted: u64,
+ finished: bool,
+}
+
+impl StreamNormalizer {
+ pub fn new(format: CaptureFormat) -> Result<Self> {
+ let format = format.validate()?;
+ let resampler = if format.sample_rate == 16_000 {
+ None
+ } else {
+ Some(
+ Fft::<f32>::new_custom(
+ format.sample_rate as usize,
+ 16000,
+ 1024,
+ 1,
+ 1,
+ WindowFunction::BlackmanHarris2,
+ FixedSync::Input,
+ )
+ .map_err(|_| VoiceError::InvalidAudio)?,
+ )
+ };
+ let delay = resampler.as_ref().map_or(0, Resampler::output_delay);
+ Ok(Self {
+ format,
+ resampler,
+ input: Vec::with_capacity(2048),
+ frame: Vec::with_capacity(VAD_FRAME_SAMPLES),
+ delay,
+ input_frames: 0,
+ emitted: 0,
+ finished: false,
+ })
+ }
+
+ pub fn push(
+ &mut self,
+ pcm: &[u8],
+ emit: &mut impl FnMut(&[f32], usize) -> Result<()>,
+ ) -> Result<()> {
+ let frame_bytes = self.format.bytes_per_frame();
+ if self.finished || !pcm.len().is_multiple_of(frame_bytes) {
+ return Err(VoiceError::InvalidAudio);
+ }
+ for frame in pcm.chunks_exact(frame_bytes) {
+ let mono = frame
+ .as_chunks::<2>()
+ .0
+ .iter()
+ .map(|b| f32::from(i16::from_le_bytes(*b)) / 32768.0)
+ .sum::<f32>()
+ / f32::from(self.format.channels);
+ self.input_frames += 1;
+ if self.resampler.is_none() {
+ self.emit_samples(&[mono], emit)?;
+ } else {
+ self.input.push(mono);
+ if self
+ .resampler
+ .as_ref()
+ .is_some_and(|r| self.input.len() == r.input_frames_next())
+ {
+ self.process_chunk(emit)?;
+ }
+ }
+ }
+ Ok(())
+ }
+
+ pub fn finish(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {
+ if self.finished {
+ return Ok(());
+ }
+ let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);
+ while self.emitted < expected {
+ let Some(resampler) = &self.resampler else {
+ return Err(VoiceError::ProcessingFailed);
+ };
+ self.input.resize(resampler.input_frames_next(), 0.0);
+ self.process_chunk(emit)?;
+ }
+ if !self.frame.is_empty() {
+ let real = self.frame.len();
+ self.frame.resize(VAD_FRAME_SAMPLES, 0.0);
+ emit(&self.frame, real)?;
+ self.frame.clear();
+ }
+ self.finished = true;
+ Ok(())
+ }
+
+ fn process_chunk(&mut self, emit: &mut impl FnMut(&[f32], usize) -> Result<()>) -> Result<()> {
+ let resampler = self.resampler.as_mut().ok_or(VoiceError::InvalidState)?;
+ let input = InterleavedSlice::new(&self.input, 1, self.input.len())
+ .map_err(|_| VoiceError::InvalidAudio)?;
+ let output = resampler
+ .process(&input, None)
+ .map_err(|_| VoiceError::ProcessingFailed)?
+ .take_data();
+ self.input.clear();
+ let skip = self.delay.min(output.len());
+ self.delay -= skip;
+ let expected = self.input_frames * 16000 / u64::from(self.format.sample_rate);
+ let take = (expected - self.emitted).min((output.len() - skip) as u64) as usize;
+ self.emit_samples(&output[skip..skip + take], emit)
+ }
+
+ fn emit_samples(
+ &mut self,
+ mut samples: &[f32],
+ emit: &mut impl FnMut(&[f32], usize) -> Result<()>,
+ ) -> Result<()> {
+ self.emitted += samples.len() as u64;
+ while !samples.is_empty() {
+ let take = samples.len().min(VAD_FRAME_SAMPLES - self.frame.len());
+ self.frame.extend_from_slice(&samples[..take]);
+ samples = &samples[take..];
+ if self.frame.len() == VAD_FRAME_SAMPLES {
+ emit(&self.frame, VAD_FRAME_SAMPLES)?;
+ self.frame.clear();
+ }
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn normalized(rate: u32, channels: u16, frames: usize, chunk: usize) -> Vec<f32> {
+ let mut normalizer = StreamNormalizer::new(CaptureFormat {
+ sample_rate: rate,
+ channels,
+ })
+ .unwrap();
+ let pcm: Vec<u8> = (0..frames)
+ .flat_map(|i| {
+ let value = ((std::f32::consts::TAU * 1000.0 * i as f32 / rate as f32).sin()
+ * 10000.0) as i16;
+ (0..channels).flat_map(move |_| value.to_le_bytes())
+ })
+ .collect();
+ let mut output = Vec::new();
+ let mut emit = |frame: &[f32], real: usize| {
+ assert_eq!(frame.len(), VAD_FRAME_SAMPLES);
+ output.extend_from_slice(&frame[..real]);
+ Ok(())
+ };
+ for bytes in pcm.chunks(chunk * usize::from(channels) * 2) {
+ normalizer.push(bytes, &mut emit).unwrap();
+ }
+ normalizer.finish(&mut emit).unwrap();
+ normalizer.finish(&mut emit).unwrap();
+ assert_eq!(output.len(), frames * 16000 / rate as usize);
+ output
+ }
+
+ #[test]
+ fn arbitrary_chunks_and_stereo_preserve_duration_and_final_tail() {
+ for rate in [8000, 16000, 44100, 48000] {
+ for frames in [1, 37, 1024, rate as usize + 19] {
+ let expected = normalized(rate, 1, frames, 1024);
+ for chunk in [1, 137, 4096] {
+ assert_eq!(
+ normalized(rate, 2, frames, chunk),
+ expected,
+ "{rate} Hz, {frames} frames, chunks {chunk}"
+ );
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn fft_delay_does_not_shift_source_timestamps() {
+ let output = normalized(48000, 1, 48000, 257);
+ let error = output[1600..14400]
+ .iter()
+ .enumerate()
+ .map(|(i, value)| {
+ let expected = (std::f32::consts::TAU * 1000.0 * (i + 1600) as f32 / 16000.0).sin()
+ * 10000.0
+ / 32768.0;
+ (value - expected).abs()
+ })
+ .fold(0.0_f32, f32::max);
+ assert!(error < 0.001, "{error}");
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rs
new file mode 100644
index 0000000000..cb8eb41377
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/recorded.rs
@@ -0,0 +1,214 @@
+use std::time::Duration;
+
+use transcribe_rs::vad::Vad;
+
+use super::{LiveControl, LiveEvent, LiveTranscription, StreamNormalizer, Transcriber};
+use crate::voice::{Result, VoiceError};
+
+const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;
+
+#[derive(Default)]
+pub(super) struct RecordedPcm(Vec<u8>);
+
+impl RecordedPcm {
+ pub fn push(&mut self, samples: &[f32]) -> Result<()> {
+ if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(self.0.len() / 2) {
+ return Err(VoiceError::BacklogFull);
+ }
+ self.0.extend(samples.iter().flat_map(|sample| {
+ ((sample * 32768.0).round().clamp(-32768.0, 32767.0) as i16).to_le_bytes()
+ }));
+ Ok(())
+ }
+
+ pub fn bytes(&self) -> &[u8] {
+ &self.0
+ }
+}
+
+impl Drop for RecordedPcm {
+ fn drop(&mut self) {
+ use zeroize::Zeroize;
+ self.0.zeroize();
+ }
+}
+
+impl LiveTranscription {
+ pub fn run_recorded(
+ &self,
+ transcriber: &Transcriber,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ self.run_recorded_with(
+ || transcriber.live_vad(),
+ |pcm| {
+ transcriber
+ .transcribe(16_000, pcm)
+ .map_err(|_| VoiceError::ProcessingFailed)
+ },
+ on_event,
+ )
+ }
+
+ fn run_recorded_with(
+ &self,
+ make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,
+ decode: impl FnOnce(&[u8]) -> Result<String>,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ let _capture = self.start()?;
+ let mut vad = make_vad()?;
+ let mut normalizer = StreamNormalizer::new(self.queue.format())?;
+ self.emit(LiveEvent::Ready, on_event)?;
+ let mut pcm = RecordedPcm::default();
+ let mut accepted = 0_u64;
+ let mut heard_speech = false;
+ let mut notified_stop = false;
+ loop {
+ self.check()?;
+ let chunk = self.queue.take();
+ let had_chunk = chunk.is_some();
+ let mut emit = |frame: &[f32], real: usize| -> Result<()> {
+ let voiced = vad
+ .is_speech(frame)
+ .map_err(|_| VoiceError::ProcessingFailed)?;
+ heard_speech |= voiced;
+ let start = accepted;
+ accepted += real as u64;
+ self.emit(
+ LiveEvent::Audio {
+ start,
+ end: accepted,
+ voiced,
+ },
+ on_event,
+ )?;
+ pcm.push(&frame[..real])
+ };
+ if let Some(chunk) = chunk {
+ normalizer.push(&chunk, &mut emit)?;
+ }
+ let drained = self.queue.is_drained();
+ if drained {
+ normalizer.finish(&mut emit)?;
+ }
+ if self.queue.is_closed() && !notified_stop {
+ self.emit(LiveEvent::CaptureClosed, on_event)?;
+ notified_stop = true;
+ }
+ if drained {
+ break;
+ }
+ if !had_chunk {
+ self.emit(LiveEvent::Idle, on_event)?;
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ }
+ drop(vad);
+ drop(normalizer);
+ self.check()?;
+ if heard_speech {
+ let text = decode(pcm.bytes())?;
+ self.check()?;
+ if !text.trim().is_empty() {
+ self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;
+ }
+ }
+ self.emit(LiveEvent::Coverage(accepted), on_event)?;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::voice::CaptureFormat;
+ use transcribe_rs::vad::EnergyVad;
+
+ #[test]
+ fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {
+ let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);
+ pcm.push(&[0.25]).unwrap();
+ assert_eq!(pcm.bytes().len(), MAX_RECORDED_SAMPLES * 2);
+ assert_eq!(pcm.push(&[0.5]), Err(VoiceError::BacklogFull));
+ assert_eq!(
+ &pcm.bytes()[pcm.bytes().len() - 2..],
+ &8192_i16.to_le_bytes()
+ );
+ }
+
+ #[test]
+ fn recorded_text_arrives_once_after_close_with_partial_tail() {
+ let stream = LiveTranscription::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap();
+ stream.feed(&[0, 32].repeat(1001)).unwrap();
+ stream.finish();
+ let mut closed = false;
+ let mut commits = Vec::new();
+ let mut covered = 0;
+ stream
+ .run_recorded_with(
+ || {
+ Ok(Box::new(EnergyVad::new(
+ super::super::VAD_FRAME_SAMPLES,
+ 0.01,
+ )))
+ },
+ |pcm| {
+ assert_eq!(pcm.len(), 1001 * 2);
+ assert_eq!(pcm, [0, 32].repeat(1001));
+ Ok("dictated words".into())
+ },
+ &mut |event| {
+ match event {
+ LiveEvent::CaptureClosed => closed = true,
+ LiveEvent::ReplaceTranscript(text) => {
+ assert!(closed);
+ commits.push(text);
+ }
+ LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),
+ LiveEvent::Coverage(end) => covered = end,
+ _ => {}
+ }
+ Ok(LiveControl::Continue)
+ },
+ )
+ .unwrap();
+ assert_eq!(commits, ["dictated words"]);
+ assert_eq!(covered, 1001);
+ }
+
+ #[test]
+ fn cancellation_during_batch_decode_does_not_publish_text() {
+ let stream = LiveTranscription::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap();
+ stream.feed(&[0, 32].repeat(512)).unwrap();
+ stream.finish();
+ let result = stream.run_recorded_with(
+ || {
+ Ok(Box::new(EnergyVad::new(
+ super::super::VAD_FRAME_SAMPLES,
+ 0.01,
+ )))
+ },
+ |_| {
+ stream.cancel();
+ Ok("late result".into())
+ },
+ &mut |event| {
+ assert!(!matches!(
+ event,
+ LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)
+ ));
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert_eq!(result, Err(VoiceError::Cancelled));
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rs
new file mode 100644
index 0000000000..4e1a0e5df0
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/session.rs
@@ -0,0 +1,166 @@
+use super::{LiveControl, LiveEvent};
+use crate::voice;
+
+#[derive(Default)]
+pub struct DictationState {
+ pub session: voice::VoiceSession,
+ text: String,
+ hypothesis: String,
+ last_commit_end: u64,
+}
+impl DictationState {
+ pub fn new(live: bool) -> Self {
+ Self {
+ session: if live {
+ voice::VoiceSession::default()
+ } else {
+ voice::VoiceSession::recorded()
+ },
+ ..Self::default()
+ }
+ }
+
+ pub fn text(&self) -> &str {
+ &self.text
+ }
+
+ pub fn hypothesis(&self) -> String {
+ format!(
+ "{}{}",
+ voice::boundary_separator(&self.text, &self.hypothesis),
+ self.hypothesis
+ )
+ }
+
+ pub fn apply(
+ &mut self,
+ event: LiveEvent,
+ finish_if_empty: impl FnOnce() -> bool,
+ ) -> voice::Result<LiveControl> {
+ if !self.session.worker_active() || self.session.phase() == voice::VoicePhase::Closed {
+ return Ok(LiveControl::Cancel);
+ }
+ match event {
+ LiveEvent::Ready => {}
+ LiveEvent::Audio { start, end, voiced } => {
+ if self.session.stop_reason().is_some() {
+ self.session.observe_drained_audio(start, end)?;
+ } else if let Some(reason) =
+ self.session
+ .observe_audio(start, end, voiced.then_some(end))?
+ {
+ self.session.stop(reason)?;
+ return Ok(LiveControl::Finish);
+ }
+ }
+ LiveEvent::Hypothesis(text) => {
+ self.session.observe_recognition(!text.trim().is_empty())?;
+ self.hypothesis = text;
+ }
+ LiveEvent::Commit { start, end, text } => {
+ if end <= start || start < self.last_commit_end {
+ return Err(voice::VoiceError::InvalidAudio);
+ }
+ self.last_commit_end = end;
+ self.session.observe_recognition(!text.trim().is_empty())?;
+ self.text
+ .push_str(voice::boundary_separator(&self.text, &text));
+ self.text.push_str(&text);
+ self.hypothesis.clear();
+ }
+ LiveEvent::ReplaceTranscript(text) => {
+ if text.trim().is_empty() {
+ return Err(voice::VoiceError::IncompleteAudio);
+ }
+ self.session.observe_recognition(true)?;
+ self.text = text;
+ self.hypothesis.clear();
+ self.last_commit_end = u64::MAX;
+ }
+ LiveEvent::Deferred => self.session.defer_recognition(),
+ LiveEvent::Coverage(end) => self.session.recognize_through(end)?,
+ LiveEvent::Idle => {
+ if self.session.silence_elapsed() && finish_if_empty() {
+ self.session.stop(voice::StopReason::Silence)?;
+ }
+ }
+ LiveEvent::CaptureClosed => {
+ if self.session.stop_reason().is_none() {
+ self.session.stop(voice::StopReason::Silence)?;
+ }
+ }
+ }
+ Ok(LiveControl::Continue)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn event(state: &mut DictationState, event: LiveEvent) {
+ state.apply(event, || true).unwrap();
+ }
+
+ #[test]
+ fn queued_audio_is_processed_before_silence_can_stop_capture() {
+ let mut state = DictationState::new(false);
+ state.session.begin_capture().unwrap();
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: 0,
+ end: 16000,
+ voiced: true,
+ },
+ );
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: 16000,
+ end: 64000,
+ voiced: false,
+ },
+ );
+ state.apply(LiveEvent::Idle, || false).unwrap();
+ assert!(state.session.stop_reason().is_none());
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: 64000,
+ end: 65000,
+ voiced: true,
+ },
+ );
+ event(&mut state, LiveEvent::Idle);
+ assert!(state.session.stop_reason().is_none());
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: 65000,
+ end: 113000,
+ voiced: false,
+ },
+ );
+ event(&mut state, LiveEvent::Idle);
+ assert_eq!(
+ state.session.stop_reason(),
+ Some(voice::StopReason::Silence)
+ );
+ }
+
+ #[test]
+ fn preview_preserves_native_spaces_and_unspaced_scripts() {
+ for (stable, partial, expected) in [
+ ("明", "天", "明天"),
+ ("hello", "world", "hello world"),
+ ("hello ", "world", "hello world"),
+ ] {
+ let mut state = DictationState::new(true);
+ state.session.begin_capture().unwrap();
+ event(&mut state, LiveEvent::ReplaceTranscript(stable.into()));
+ event(&mut state, LiveEvent::Hypothesis(partial.into()));
+ assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);
+ }
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/text.rs b/rust/crates/ensu/src/transcription/text.rs
deleted file mode 100644
index 781fbbd952..0000000000
--- a/rust/crates/ensu/src/transcription/text.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-use regex::{Regex, regex};
-use std::sync::LazyLock;
-
-const FILLER_WORDS: &[&str] = &[
- "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha",
- "ehh",
-];
-
-#[expect(
- clippy::expect_used,
- reason = "Filler words are escaped before insertion into the fixed regex"
-)]
-static FILLER_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
- FILLER_WORDS
- .iter()
- .map(|word| {
- Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word)))
- .expect("escaped filler word forms a valid regex")
- })
- .collect()
-});
-
-pub fn filter_transcription_output(text: &str) -> String {
- let mut filtered = text.to_string();
-
- for pattern in FILLER_PATTERNS.iter() {
- filtered = pattern.replace_all(&filtered, "").to_string();
- }
-
- filtered = collapse_stutters(&filtered);
- filtered = regex!(r"\s{2,}").replace_all(&filtered, " ").to_string();
- filtered.trim().to_string()
-}
-
-fn collapse_stutters(text: &str) -> String {
- let words = text.split_whitespace().collect::<Vec<_>>();
- if words.is_empty() {
- return text.to_string();
- }
-
- let mut result = Vec::new();
- let mut i = 0;
-
- while i < words.len() {
- let word = words[i];
- let word_lower = word.to_lowercase();
-
- if word_lower.len() <= 2 && word_lower.chars().all(char::is_alphabetic) {
- let mut count = 1;
- while i + count < words.len() && words[i + count].to_lowercase() == word_lower {
- count += 1;
- }
-
- result.push(word);
- i += if count >= 3 { count } else { 1 };
- } else {
- result.push(word);
- i += 1;
- }
- }
-
- result.join(" ")
-}
-
-#[cfg(test)]
-mod tests {
- use super::filter_transcription_output;
-
- #[test]
- fn removes_filler_words() {
- assert_eq!(
- filter_transcription_output("um hello uh world"),
- "hello world"
- );
- }
-
- #[test]
- fn collapses_repeated_short_stutters() {
- assert_eq!(filter_transcription_output("I I I think"), "I think");
- }
-}
diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rs
index cbfd111cef..16a34ba8ee 100644
--- a/rust/crates/ensu/src/transcription/transcriber.rs
+++ b/rust/crates/ensu/src/transcription/transcriber.rs
@@ -3,31 +3,105 @@ use std::sync::{Mutex, MutexGuard, PoisonError};
use transcribe_rs::onnx::Quantization;
use transcribe_rs::onnx::parakeet::{ParakeetModel, ParakeetParams, TimestampGranularity};
+use transcribe_rs::{OrtAccelerator, get_ort_accelerator};
+use super::graph_cache::{GRAPHS, GraphCache, LoadPlan, SOURCES};
use crate::transcription::audio::extract_speech_from_pcm16;
-use crate::transcription::text::filter_transcription_output;
use crate::transcription::{Result, TranscriptionError};
pub struct Transcriber {
model_dir: PathBuf,
vad_model_path: PathBuf,
loaded: Mutex<Option<ParakeetModel>>,
+ graph_cache: Option<GraphCache>,
}
impl Transcriber {
+ pub(crate) fn live_vad(&self) -> crate::voice::Result<Box<dyn transcribe_rs::vad::Vad>> {
+ super::SileroV4Vad::new(&self.vad_model_path)
+ .map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>)
+ .map_err(|_| crate::voice::VoiceError::ProcessingFailed)
+ }
+
+ pub(crate) fn transcribe_window(
+ &self,
+ samples: &[f32],
+ ) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {
+ use crate::voice::{AlignedWord, MAX_WINDOW_SAMPLES, VoiceError};
+ if samples.is_empty()
+ || samples.len() > MAX_WINDOW_SAMPLES
+ || samples.iter().any(|s| !s.is_finite())
+ {
+ return Err(VoiceError::InvalidAudio);
+ }
+ let mut loaded = self.lock();
+ let model = ensure_loaded(
+ &mut loaded,
+ &self.model_dir,
+ self.graph_cache.as_ref(),
+ true,
+ )
+ .map_err(|_| VoiceError::ProcessingFailed)?;
+ let result = model
+ .transcribe_with(
+ samples,
+ &ParakeetParams {
+ timestamp_granularity: Some(TimestampGranularity::Word),
+ ..Default::default()
+ },
+ )
+ .map_err(|_| VoiceError::ProcessingFailed)?;
+ result
+ .segments
+ .ok_or(VoiceError::UnstableTranscript)?
+ .into_iter()
+ .map(|word| {
+ if !word.start.is_finite()
+ || !word.end.is_finite()
+ || word.end < word.start
+ || word.start * 16000.0 > samples.len() as f32
+ {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ Ok(AlignedWord {
+ text: word.text,
+ start: (word.start.max(0.0) * 16000.0).round() as u64,
+ end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples.len() as u64),
+ })
+ })
+ .collect()
+ }
+
pub fn new(model_dir: impl Into<PathBuf>, vad_model_path: impl Into<PathBuf>) -> Self {
Self {
model_dir: model_dir.into(),
vad_model_path: vad_model_path.into(),
loaded: Mutex::new(None),
+ graph_cache: None,
}
}
+ pub fn with_cache(
+ model_dir: impl Into<PathBuf>,
+ vad_model_path: impl Into<PathBuf>,
+ cache_directory: impl Into<PathBuf>,
+ device_identity: String,
+ ) -> Self {
+ let mut transcriber = Self::new(model_dir, vad_model_path);
+ transcriber.graph_cache = Some(GraphCache::new(cache_directory.into(), device_identity));
+ transcriber
+ }
+
pub fn load_model(&self) -> Result<()> {
if !self.model_dir.is_dir() {
return Err(TranscriptionError::NotDownloaded);
}
- ensure_loaded(&mut self.lock(), &self.model_dir)?;
+ ensure_loaded(
+ &mut self.lock(),
+ &self.model_dir,
+ self.graph_cache.as_ref(),
+ true,
+ )?;
Ok(())
}
@@ -52,7 +126,12 @@ impl Transcriber {
}
let mut loaded = self.lock();
- let model = ensure_loaded(&mut loaded, &self.model_dir)?;
+ let model = ensure_loaded(
+ &mut loaded,
+ &self.model_dir,
+ self.graph_cache.as_ref(),
+ false,
+ )?;
let result = model.transcribe_with(
&speech,
&ParakeetParams {
@@ -61,7 +140,7 @@ impl Transcriber {
},
)?;
- Ok(filter_transcription_output(&result.text))
+ Ok(result.text.trim().to_owned())
}
fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {
@@ -72,9 +151,82 @@ impl Transcriber {
fn ensure_loaded<'a>(
loaded: &'a mut Option<ParakeetModel>,
model_dir: &Path,
+ cache: Option<&GraphCache>,
+ may_build: bool,
) -> Result<&'a mut ParakeetModel> {
match loaded {
Some(model) => Ok(model),
- slot => Ok(slot.insert(ParakeetModel::load(model_dir, &Quantization::Int8)?)),
+ slot => {
+ for name in SOURCES {
+ if !model_dir.join(name).is_file() {
+ return Err(TranscriptionError::NotDownloaded);
+ }
+ }
+ let cache = cache.filter(|_| {
+ get_ort_accelerator() == OrtAccelerator::CpuOnly
+ || OrtAccelerator::available() == [OrtAccelerator::CpuOnly]
+ });
+ let result = if let Some(cache) = cache {
+ let configuration = format!(
+ "{};cpu;parallel=true;intra=auto;threads={}",
+ ort::info(),
+ std::thread::available_parallelism()
+ .map(std::num::NonZeroUsize::get)
+ .unwrap_or(1)
+ );
+ cache.load(model_dir, &configuration, may_build, |plan| {
+ load_plan(model_dir, plan)
+ })
+ } else {
+ load_plan(model_dir, LoadPlan::Source { save_to: None })
+ };
+ Ok(slot.insert(result?))
+ }
+ }
+}
+
+fn load_plan(model_dir: &Path, plan: LoadPlan<'_>) -> Result<ParakeetModel> {
+ use ort::session::builder::GraphOptimizationLevel;
+ let directory = match plan {
+ LoadPlan::Source { save_to: None } => model_dir,
+ LoadPlan::Cached { directory } => directory,
+ LoadPlan::Source {
+ save_to: Some(directory),
+ } => {
+ for name in GRAPHS {
+ let session = ort::session::Session::builder()?
+ .with_optimization_level(GraphOptimizationLevel::Level3)?
+ .with_optimized_model_path(directory.join(name))?
+ .with_parallel_execution(true)?
+ .with_execution_providers([ort::ep::CPU::default().build()])?
+ .commit_from_file(model_dir.join(name))?;
+ drop(session);
+ }
+ directory
+ }
+ };
+ Ok(ParakeetModel::load(directory, &Quantization::Int8)?)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn missing_quantized_graph_never_falls_back_to_full_precision() {
+ let directory = tempfile::tempdir().unwrap();
+ for name in [
+ "encoder-model.onnx",
+ "decoder_joint-model.onnx",
+ "nemo128.onnx",
+ "vocab.txt",
+ ] {
+ std::fs::write(directory.path().join(name), b"not a model").unwrap();
+ }
+ let transcriber = Transcriber::new(directory.path(), directory.path().join("vad.onnx"));
+ assert!(matches!(
+ transcriber.load_model(),
+ Err(TranscriptionError::NotDownloaded)
+ ));
}
}
diff --git a/rust/crates/ensu/src/transcription/vad.rs b/rust/crates/ensu/src/transcription/vad.rs
new file mode 100644
index 0000000000..8869d0139c
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/vad.rs
@@ -0,0 +1,106 @@
+use ort::{
+ inputs,
+ session::{Session, builder::GraphOptimizationLevel},
+ value::TensorRef,
+};
+use std::path::Path;
+use transcribe_rs::{TranscribeError, vad::Vad};
+
+pub(super) const SILERO_V4_FRAME_SAMPLES: usize = 512;
+
+pub(crate) struct SileroV4Vad {
+ session: Session,
+ hidden: [f32; 128],
+ cell: [f32; 128],
+}
+
+fn inference(error: impl std::fmt::Display) -> TranscribeError {
+ TranscribeError::Inference(format!("Silero v4: {error}"))
+}
+
+impl SileroV4Vad {
+ pub fn new(path: impl AsRef<Path>) -> Result<Self, TranscribeError> {
+ let session = Session::builder()
+ .map_err(inference)?
+ .with_optimization_level(GraphOptimizationLevel::Level3)
+ .map_err(inference)?
+ .with_intra_threads(1)
+ .map_err(inference)?
+ .with_inter_threads(1)
+ .map_err(inference)?
+ .commit_from_file(path)
+ .map_err(inference)?;
+ Ok(Self {
+ session,
+ hidden: [0.0; 128],
+ cell: [0.0; 128],
+ })
+ }
+
+ fn speech_probability(&mut self, frame: &[f32]) -> Result<f32, TranscribeError> {
+ validate_frame(frame)?;
+ let sample_rate = [16000_i64];
+ let outputs = self.session.run(inputs![
+ "input" => TensorRef::from_array_view(([1, SILERO_V4_FRAME_SAMPLES], frame)).map_err(inference)?,
+ "sr" => TensorRef::from_array_view(([1], sample_rate.as_slice())).map_err(inference)?,
+ "h" => TensorRef::from_array_view(([2,1,64], self.hidden.as_slice())).map_err(inference)?,
+ "c" => TensorRef::from_array_view(([2,1,64], self.cell.as_slice())).map_err(inference)?,
+ ]).map_err(inference)?;
+ let (shape, probability) = outputs
+ .get("output")
+ .ok_or_else(|| inference("missing probability"))?
+ .try_extract_tensor::<f32>()
+ .map_err(inference)?;
+ if shape.as_ref() != [1, 1]
+ || probability.len() != 1
+ || !probability[0].is_finite()
+ || !(0.0..=1.0).contains(&probability[0])
+ {
+ return Err(inference("invalid probability"));
+ }
+ let probability = probability[0];
+ let (h_shape, hidden) = outputs
+ .get("hn")
+ .ok_or_else(|| inference("missing hidden state"))?
+ .try_extract_tensor::<f32>()
+ .map_err(inference)?;
+ let (c_shape, cell) = outputs
+ .get("cn")
+ .ok_or_else(|| inference("missing cell state"))?
+ .try_extract_tensor::<f32>()
+ .map_err(inference)?;
+ if h_shape.as_ref() != [2, 1, 64]
+ || c_shape.as_ref() != [2, 1, 64]
+ || hidden.len() != 128
+ || cell.len() != 128
+ || hidden.iter().chain(cell).any(|x| !x.is_finite())
+ {
+ return Err(inference("invalid recurrent state"));
+ }
+ self.hidden.copy_from_slice(hidden);
+ self.cell.copy_from_slice(cell);
+ Ok(probability)
+ }
+}
+
+fn validate_frame(frame: &[f32]) -> Result<(), TranscribeError> {
+ if frame.len() != SILERO_V4_FRAME_SAMPLES || frame.iter().any(|s| !s.is_finite()) {
+ return Err(TranscribeError::Audio(
+ "Silero v4 requires 512 finite samples at 16 kHz".into(),
+ ));
+ }
+ Ok(())
+}
+
+impl Vad for SileroV4Vad {
+ fn frame_size(&self) -> usize {
+ SILERO_V4_FRAME_SAMPLES
+ }
+ fn is_speech(&mut self, frame: &[f32]) -> Result<bool, TranscribeError> {
+ Ok(self.speech_probability(frame)? > 0.5)
+ }
+ fn reset(&mut self) {
+ self.hidden.fill(0.0);
+ self.cell.fill(0.0);
+ }
+}
diff --git a/rust/crates/ensu/src/voice/audio.rs b/rust/crates/ensu/src/voice/audio.rs
new file mode 100644
index 0000000000..5ee6a89eb2
--- /dev/null
+++ b/rust/crates/ensu/src/voice/audio.rs
@@ -0,0 +1,121 @@
+use std::collections::VecDeque;
+
+use super::{Result, VoiceError};
+
+pub const VOICE_SAMPLE_RATE: u64 = 16_000;
+pub const MAX_WINDOW_SAMPLES: usize = 10 * VOICE_SAMPLE_RATE as usize;
+pub const MAX_RING_SAMPLES: usize = 30 * VOICE_SAMPLE_RATE as usize;
+pub const MAX_BACKLOG_SAMPLES: u64 = 10 * VOICE_SAMPLE_RATE;
+
+#[derive(Debug)]
+pub struct AudioRing {
+ samples: VecDeque<f32>,
+ start: u64,
+ processed: u64,
+}
+
+impl Default for AudioRing {
+ fn default() -> Self {
+ Self {
+ samples: VecDeque::with_capacity(MAX_RING_SAMPLES),
+ start: 0,
+ processed: 0,
+ }
+ }
+}
+
+impl AudioRing {
+ pub fn push(&mut self, offset: u64, samples: &[f32]) -> Result<()> {
+ if offset != self.accepted() || samples.iter().any(|v| !v.is_finite()) {
+ return Err(VoiceError::InvalidAudio);
+ }
+ let end = offset
+ .checked_add(samples.len() as u64)
+ .ok_or(VoiceError::InvalidAudio)?;
+ if end - self.processed > MAX_BACKLOG_SAMPLES
+ || self.samples.len().saturating_add(samples.len()) > MAX_RING_SAMPLES
+ {
+ return Err(VoiceError::BacklogFull);
+ }
+ self.samples.extend(samples);
+ Ok(())
+ }
+
+ pub fn window(&self, start: u64, end: u64) -> Result<Vec<f32>> {
+ if start < self.start
+ || end > self.accepted()
+ || end < start
+ || end - start > MAX_WINDOW_SAMPLES as u64
+ {
+ return Err(VoiceError::InvalidAudio);
+ }
+ Ok(self
+ .samples
+ .iter()
+ .skip((start - self.start) as usize)
+ .take((end - start) as usize)
+ .copied()
+ .collect())
+ }
+
+ pub fn acknowledge(&mut self, processed: u64, retain_from: u64) -> Result<()> {
+ if processed < self.processed
+ || processed > self.accepted()
+ || retain_from < self.start
+ || retain_from > processed
+ {
+ return Err(VoiceError::InvalidAudio);
+ }
+ self.samples.drain(..(retain_from - self.start) as usize);
+ self.start = retain_from;
+ self.processed = processed;
+ Ok(())
+ }
+
+ pub fn accepted(&self) -> u64 {
+ self.start + self.samples.len() as u64
+ }
+ pub fn retained_from(&self) -> u64 {
+ self.start
+ }
+ pub fn processed(&self) -> u64 {
+ self.processed
+ }
+ #[cfg(test)]
+ pub(super) fn retained_samples(&self) -> usize {
+ self.samples.len()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn saturation_never_overwrites_unrecognized_audio() {
+ let mut ring = AudioRing::default();
+ ring.push(0, &vec![0.25; MAX_BACKLOG_SAMPLES as usize])
+ .unwrap();
+ assert_eq!(
+ ring.push(ring.accepted(), &[0.5]),
+ Err(VoiceError::BacklogFull)
+ );
+ assert_eq!(ring.accepted(), MAX_BACKLOG_SAMPLES);
+ assert_eq!(ring.window(0, 1).unwrap(), vec![0.25]);
+ ring.acknowledge(ring.accepted(), ring.accepted() - 8000)
+ .unwrap();
+ ring.push(ring.accepted(), &[0.5]).unwrap();
+ assert_eq!(ring.retained_samples(), 8001);
+ assert_eq!(ring.accepted(), MAX_BACKLOG_SAMPLES + 1);
+ }
+
+ #[test]
+ fn invalid_offsets_and_nan_are_rejected_atomically() {
+ let mut ring = AudioRing::default();
+ assert_eq!(ring.push(1, &[0.0]), Err(VoiceError::InvalidAudio));
+ assert_eq!(ring.push(0, &[f32::NAN]), Err(VoiceError::InvalidAudio));
+ assert_eq!(ring.accepted(), 0);
+ ring.push(0, &[0.1, 0.2]).unwrap();
+ assert_eq!(ring.acknowledge(3, 0), Err(VoiceError::InvalidAudio));
+ }
+}
diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rs
new file mode 100644
index 0000000000..544e8941a3
--- /dev/null
+++ b/rust/crates/ensu/src/voice/capture.rs
@@ -0,0 +1,185 @@
+use std::collections::VecDeque;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Mutex, MutexGuard, PoisonError};
+
+use super::{Result, VoiceError};
+
+#[derive(Clone, Copy, Debug)]
+pub struct CaptureFormat {
+ pub sample_rate: u32,
+ pub channels: u16,
+}
+
+impl CaptureFormat {
+ pub fn validate(self) -> Result<Self> {
+ if !(8_000..=48_000).contains(&self.sample_rate) || !matches!(self.channels, 1 | 2) {
+ return Err(VoiceError::InvalidAudio);
+ }
+ Ok(self)
+ }
+
+ pub fn bytes_per_frame(self) -> usize {
+ usize::from(self.channels) * 2
+ }
+}
+
+#[derive(Default)]
+struct Pending {
+ chunks: VecDeque<Vec<u8>>,
+ bytes: usize,
+ closed: bool,
+}
+
+pub struct CaptureQueue {
+ format: CaptureFormat,
+ pending: Mutex<Pending>,
+ failed: AtomicBool,
+ cancelled: AtomicBool,
+ submitted_frames: AtomicU64,
+}
+
+impl CaptureQueue {
+ pub fn new(format: CaptureFormat) -> Result<Self> {
+ Ok(Self {
+ format: format.validate()?,
+ pending: Mutex::new(Pending::default()),
+ failed: AtomicBool::new(false),
+ cancelled: AtomicBool::new(false),
+ submitted_frames: AtomicU64::new(0),
+ })
+ }
+
+ pub fn format(&self) -> CaptureFormat {
+ self.format
+ }
+
+ pub fn push(&self, pcm: &[u8]) -> Result<()> {
+ if self.cancelled.load(Ordering::Acquire) {
+ return Err(VoiceError::Cancelled);
+ }
+ if self.failed.load(Ordering::Acquire) {
+ return Err(VoiceError::BacklogFull);
+ }
+ let frame_bytes = self.format.bytes_per_frame();
+ if pcm.is_empty()
+ || !pcm.len().is_multiple_of(frame_bytes)
+ || pcm.len() / frame_bytes > self.format.sample_rate as usize / 5
+ {
+ self.failed.store(true, Ordering::Release);
+ return Err(VoiceError::InvalidAudio);
+ }
+ let owned = pcm.to_vec();
+ let mut pending = self.lock();
+ if pending.closed {
+ return Err(VoiceError::InvalidState);
+ }
+ self.submitted_frames
+ .fetch_add((pcm.len() / frame_bytes) as u64, Ordering::AcqRel);
+ if pending.bytes + pcm.len() > self.format.sample_rate as usize * frame_bytes * 2
+ || pending.chunks.len() >= 128
+ {
+ self.failed.store(true, Ordering::Release);
+ return Err(VoiceError::BacklogFull);
+ }
+ pending.bytes += pcm.len();
+ pending.chunks.push_back(owned);
+ Ok(())
+ }
+
+ pub fn take(&self) -> Option<Vec<u8>> {
+ let mut pending = self.lock();
+ let chunk = pending.chunks.pop_front()?;
+ pending.bytes -= chunk.len();
+ Some(chunk)
+ }
+
+ pub fn finish(&self) {
+ self.lock().closed = true;
+ }
+
+ pub fn finish_if_empty(&self) -> bool {
+ let mut pending = self.lock();
+ if !pending.chunks.is_empty() {
+ return false;
+ }
+ pending.closed = true;
+ true
+ }
+
+ pub fn cancel(&self) {
+ self.cancelled.store(true, Ordering::Release);
+ self.finish();
+ }
+
+ pub fn is_cancelled(&self) -> bool {
+ self.cancelled.load(Ordering::Acquire)
+ }
+
+ pub fn failed(&self) -> bool {
+ self.failed.load(Ordering::Acquire)
+ }
+
+ pub fn is_drained(&self) -> bool {
+ let pending = self.lock();
+ pending.closed && pending.chunks.is_empty()
+ }
+
+ pub fn is_closed(&self) -> bool {
+ self.lock().closed
+ }
+
+ pub fn submitted_frames(&self) -> u64 {
+ self.submitted_frames.load(Ordering::Acquire)
+ }
+
+ pub fn clear(&self) {
+ let mut pending = self.lock();
+ pending.closed = true;
+ pending.chunks.clear();
+ pending.bytes = 0;
+ }
+
+ fn lock(&self) -> MutexGuard<'_, Pending> {
+ self.pending.lock().unwrap_or_else(PoisonError::into_inner)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn queue_bounds_bytes_and_saturation_is_sticky() {
+ let queue = CaptureQueue::new(CaptureFormat {
+ sample_rate: 48000,
+ channels: 2,
+ })
+ .unwrap();
+ let chunk = vec![7; 48000 / 5 * 4];
+ for _ in 0..10 {
+ queue.push(&chunk).unwrap();
+ }
+ assert_eq!(queue.push(&chunk), Err(VoiceError::BacklogFull));
+ assert_eq!(queue.lock().bytes, 96000 * 4);
+ assert_eq!(queue.take().unwrap(), chunk);
+ assert!(queue.failed());
+ assert_eq!(queue.push(&chunk), Err(VoiceError::BacklogFull));
+ }
+
+ #[test]
+ fn countdown_cannot_skip_queued_speech_and_stop_drains_tail() {
+ let queue = CaptureQueue::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap();
+ queue.push(&[1, 0]).unwrap();
+ assert!(!queue.finish_if_empty());
+ assert_eq!(queue.take(), Some(vec![1, 0]));
+ assert!(queue.finish_if_empty());
+ assert!(queue.is_drained());
+ assert_eq!(queue.push(&[1, 0]), Err(VoiceError::InvalidState));
+ queue.cancel();
+ assert_eq!(queue.push(&[1, 0]), Err(VoiceError::Cancelled));
+ }
+}
diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rs
new file mode 100644
index 0000000000..983d956d39
--- /dev/null
+++ b/rust/crates/ensu/src/voice/composer.rs
@@ -0,0 +1,265 @@
+use std::ops::Range;
+
+use super::{Result, VoiceError};
+
+pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {
+ if start > end {
+ return Err(VoiceError::InvalidTextRange);
+ }
+ let mut units = 0_u32;
+ let mut from = None;
+ for (byte, character) in text
+ .char_indices()
+ .chain(std::iter::once((text.len(), '\0')))
+ {
+ if units == start {
+ from = Some(byte);
+ }
+ if units == end {
+ return Ok(from.ok_or(VoiceError::InvalidTextRange)?..byte);
+ }
+ units = units
+ .checked_add(character.len_utf16() as u32)
+ .ok_or(VoiceError::InvalidTextRange)?;
+ }
+ Err(VoiceError::InvalidTextRange)
+}
+
+pub fn boundary_separator(left: &str, right: &str) -> &'static str {
+ fn unspaced(c: char) -> bool {
+ matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)
+ }
+ let left_edge = left
+ .chars()
+ .rev()
+ .find(|c| {
+ !regex::regex!(r"\A[\p{M}\p{Join_Control}]\z").is_match(c.encode_utf8(&mut [0; 4]))
+ })
+ .or_else(|| left.chars().next());
+ let right_edge = right.chars().next();
+ match (left_edge, right_edge) {
+ (Some(a), Some(b))
+ if (a.is_alphanumeric()
+ || matches!(
+ a,
+ '.' | '!' | '?' | ':' | ';' | ',' | '।' | '॥' | '؟' | '؛' | '،'
+ ))
+ && b.is_alphanumeric()
+ && !unspaced(a)
+ && !unspaced(b) =>
+ {
+ " "
+ }
+ _ => "",
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ComposerAnchor {
+ pub destination: String,
+ pub revision: u64,
+ pub start_utf16: u32,
+ pub end_utf16: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct VoiceInsertion {
+ pub anchor: ComposerAnchor,
+ pub text: String,
+}
+
+pub fn preview_insertion(
+ text: &str,
+ anchor: &ComposerAnchor,
+ voice_text: &str,
+) -> Result<(String, u32)> {
+ let range = utf16_range(text, anchor.start_utf16, anchor.end_utf16)?;
+ if voice_text.is_empty() {
+ return Ok((text.to_owned(), anchor.end_utf16));
+ }
+ let before = &text[..range.start];
+ let after = &text[range.end..];
+ let prefix = boundary_separator(before, voice_text);
+ let suffix = boundary_separator(voice_text, after);
+ let caret = before.encode_utf16().count()
+ + prefix.encode_utf16().count()
+ + voice_text.encode_utf16().count()
+ + suffix.encode_utf16().count();
+ Ok((
+ format!("{before}{prefix}{voice_text}{suffix}{after}"),
+ caret as u32,
+ ))
+}
+
+#[derive(Debug, Default)]
+pub struct ComposerTransfer {
+ receipt: Option<(VoiceInsertion, u32)>,
+}
+
+impl ComposerTransfer {
+ pub fn apply(
+ &mut self,
+ insertion: &VoiceInsertion,
+ destination: &str,
+ revision: &mut u64,
+ text: &mut String,
+ foreground: bool,
+ composing: bool,
+ ) -> Result<u32> {
+ if let Some((applied, receipt)) = &self.receipt {
+ return if applied == insertion {
+ Ok(*receipt)
+ } else {
+ Err(VoiceError::RevisionConflict)
+ };
+ }
+ if !foreground || insertion.anchor.destination != destination {
+ return Err(VoiceError::RevisionConflict);
+ }
+ if composing {
+ return Err(VoiceError::PendingEdit);
+ }
+ if insertion.text.trim().is_empty() {
+ return Err(VoiceError::InvalidState);
+ }
+ let mut anchor = insertion.anchor.clone();
+ if anchor.revision != *revision {
+ anchor.start_utf16 = text.encode_utf16().count() as u32;
+ anchor.end_utf16 = anchor.start_utf16;
+ }
+ let (rendered, caret) = preview_insertion(text, &anchor, &insertion.text)?;
+ *text = rendered;
+ *revision += 1;
+ self.receipt = Some((insertion.clone(), caret));
+ Ok(caret)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn insertion(start: u32, end: u32, text: &str) -> VoiceInsertion {
+ VoiceInsertion {
+ text: text.into(),
+ anchor: ComposerAnchor {
+ destination: "chat".into(),
+ revision: 7,
+ start_utf16: start,
+ end_utf16: end,
+ },
+ }
+ }
+
+ #[test]
+ fn selected_text_is_replaced_once_even_after_lost_acknowledgement() {
+ let mut owner = ComposerTransfer::default();
+ let mut text = " meet Friday\n".to_owned();
+ let mut revision = 7;
+ let insert = insertion(7, 13, "Saturday");
+ let receipt = owner
+ .apply(&insert, "chat", &mut revision, &mut text, true, false)
+ .unwrap();
+ assert_eq!(text, " meet Saturday\n");
+ assert_eq!(
+ owner
+ .apply(
+ &insert,
+ "different-chat",
+ &mut revision,
+ &mut text,
+ false,
+ false
+ )
+ .unwrap(),
+ receipt
+ );
+ assert_eq!(revision, 8);
+ assert_eq!(text, " meet Saturday\n");
+ }
+
+ #[test]
+ fn changed_draft_appends_once_but_a_different_chat_is_rejected() {
+ let mut owner = ComposerTransfer::default();
+ let mut text = "My edited draft".to_owned();
+ let mut revision = 8;
+ let insert = insertion(0, 0, "hello");
+ for (destination, foreground, composing) in [
+ ("another-chat", true, false),
+ ("chat", false, false),
+ ("chat", true, true),
+ ] {
+ assert!(
+ owner
+ .apply(
+ &insert,
+ destination,
+ &mut revision,
+ &mut text,
+ foreground,
+ composing
+ )
+ .is_err()
+ );
+ }
+ assert_eq!(text, "My edited draft");
+ owner
+ .apply(&insert, "chat", &mut revision, &mut text, true, false)
+ .unwrap();
+ assert_eq!(text, "My edited draft hello");
+ owner
+ .apply(&insert, "chat", &mut revision, &mut text, true, false)
+ .unwrap();
+ assert_eq!(text, "My edited draft hello");
+ assert_eq!(revision, 9);
+ }
+
+ #[test]
+ fn utf16_ranges_preserve_text_and_reject_split_surrogates() {
+ for text in ["👩🏽‍💻", "e\u{301}", "कि", "क्ष"] {
+ let end = text.encode_utf16().count() as u32;
+ assert_eq!(utf16_range(text, 0, end).unwrap(), 0..text.len());
+ for (byte, character) in text.char_indices() {
+ let start = text[..byte].encode_utf16().count() as u32;
+ assert_eq!(
+ utf16_range(text, start, start + character.len_utf16() as u32).unwrap(),
+ byte..byte + character.len_utf8()
+ );
+ if character.len_utf16() == 2 {
+ assert_eq!(
+ utf16_range(text, start, start + 1),
+ Err(VoiceError::InvalidTextRange)
+ );
+ assert_eq!(
+ utf16_range(text, start + 1, end),
+ Err(VoiceError::InvalidTextRange)
+ );
+ }
+ }
+ }
+ for (start, end) in [(2, 1), (0, 9), (9, 9)] {
+ assert_eq!(
+ utf16_range("hello", start, end),
+ Err(VoiceError::InvalidTextRange)
+ );
+ }
+ assert_eq!(utf16_range("你好 مرحبا", 0, 2).unwrap(), 0..6);
+ }
+
+ #[test]
+ fn insertion_preserves_whitespace_and_script_boundaries() {
+ assert_eq!(boundary_separator("hello", "world"), " ");
+ assert_eq!(boundary_separator("你好", "世界"), "");
+ assert_eq!(boundary_separator("hello\n", "world"), "");
+ assert_eq!(boundary_separator("مرحبا", "بك"), " ");
+ assert_eq!(boundary_separator("कि", "Saturday"), " ");
+ assert_eq!(boundary_separator("क्‍", "Saturday"), " ");
+ assert_eq!(boundary_separator("cafe\u{301}", "Saturday"), " ");
+ assert_eq!(boundary_separator("你\u{301}", "好"), "");
+ assert_eq!(boundary_separator("👩🏽‍💻", "Saturday"), "");
+ assert_eq!(boundary_separator("Done.", "Next"), " ");
+ assert_eq!(boundary_separator("है।", "बैठक"), " ");
+ assert_eq!(boundary_separator("نعم؟", "موعد"), " ");
+ assert_eq!(boundary_separator("안녕", "친구"), " ");
+ }
+}
diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rs
new file mode 100644
index 0000000000..dc70c90c0c
--- /dev/null
+++ b/rust/crates/ensu/src/voice/mod.rs
@@ -0,0 +1,39 @@
+mod audio;
+mod capture;
+mod composer;
+mod rolling;
+mod session;
+
+pub use audio::*;
+pub use capture::*;
+pub use composer::*;
+pub use rolling::*;
+pub use session::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+pub enum VoiceError {
+ #[error("Audio continuity or format is invalid")]
+ InvalidAudio,
+ #[error("Audio backlog limit reached")]
+ BacklogFull,
+ #[error("Voice recording was cancelled")]
+ Cancelled,
+ #[error("Speech alignment could not be preserved")]
+ UnstableTranscript,
+ #[error("Speech processing failed")]
+ ProcessingFailed,
+ #[error("Voice event belongs to an inactive recording")]
+ StaleEvent,
+ #[error("Voice operation is not valid in the current state")]
+ InvalidState,
+ #[error("Text range is not on a character boundary")]
+ InvalidTextRange,
+ #[error("Text or destination changed")]
+ RevisionConflict,
+ #[error("Some speech could not be finalized")]
+ IncompleteAudio,
+ #[error("Keyboard input is still composing")]
+ PendingEdit,
+}
+
+pub type Result<T> = std::result::Result<T, VoiceError>;
diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rs
new file mode 100644
index 0000000000..ba43201173
--- /dev/null
+++ b/rust/crates/ensu/src/voice/rolling.rs
@@ -0,0 +1,511 @@
+use std::collections::VecDeque;
+
+use super::{
+ AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, boundary_separator,
+};
+
+const PRE_SPEECH: u64 = VOICE_SAMPLE_RATE;
+const PHRASE_SILENCE: u64 = VOICE_SAMPLE_RATE * 3;
+const LOOKBACK: u64 = VOICE_SAMPLE_RATE * 3 / 2;
+const UPDATE_SAMPLES: u64 = VOICE_SAMPLE_RATE / 2;
+const ALIGNMENT_TOLERANCE: u64 = VOICE_SAMPLE_RATE / 5;
+const REQUIRED_OBSERVATIONS: u8 = 3;
+const RIGHT_CONTEXT_WORDS: usize = 2;
+
+#[derive(Clone, Debug)]
+pub struct AlignedWord {
+ pub text: String,
+ pub start: u64,
+ pub end: u64,
+}
+
+pub struct DecodeWindow {
+ pub id: u64,
+ pub start: u64,
+ pub end: u64,
+ pub final_phrase: bool,
+ pub samples: Vec<f32>,
+}
+
+#[derive(Debug)]
+pub struct RecognitionUpdate {
+ pub commit: Option<(u64, u64, String)>,
+ pub hypothesis: String,
+ pub covered_through: u64,
+}
+
+struct Region {
+ start: u64,
+ last_voice: u64,
+ end: Option<u64>,
+}
+
+#[derive(Default)]
+pub struct RollingRecognizer {
+ ring: AudioRing,
+ regions: VecDeque<Region>,
+ previous: Vec<AlignedWord>,
+ observations: Vec<u8>,
+ anchor: Option<AlignedWord>,
+ covered: u64,
+ last_decoded: u64,
+ next_id: u64,
+ in_flight: Option<(u64, u64, u64, bool)>,
+}
+
+impl RollingRecognizer {
+ pub fn feed(&mut self, samples: &[f32], voiced: bool) -> Result<()> {
+ if samples.is_empty() {
+ return Err(VoiceError::InvalidAudio);
+ }
+ let start = self.ring.accepted();
+ self.ring.push(start, samples)?;
+ let end = self.ring.accepted();
+ if voiced {
+ if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none()) {
+ region.last_voice = end;
+ } else {
+ let prior_end = self
+ .regions
+ .back()
+ .and_then(|r| r.end)
+ .unwrap_or(self.covered);
+ self.regions.push_back(Region {
+ start: start
+ .saturating_sub(PRE_SPEECH)
+ .max(self.ring.retained_from())
+ .max(prior_end),
+ last_voice: end,
+ end: None,
+ });
+ }
+ } else if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none())
+ && end - region.last_voice >= PHRASE_SILENCE
+ {
+ region.end = Some(end);
+ }
+ self.release_silence()?;
+ Ok(())
+ }
+
+ fn release_silence(&mut self) -> Result<()> {
+ if self.regions.is_empty() {
+ let end = self.ring.accepted();
+ self.ring.acknowledge(
+ end,
+ end.saturating_sub(PRE_SPEECH)
+ .max(self.ring.retained_from()),
+ )?;
+ self.covered = self.covered.max(self.ring.retained_from());
+ }
+ Ok(())
+ }
+
+ pub fn finish(&mut self) {
+ if let Some(region) = self.regions.back_mut().filter(|r| r.end.is_none()) {
+ region.end = Some(self.ring.accepted());
+ }
+ }
+
+ pub fn next_window(&mut self) -> Result<Option<DecodeWindow>> {
+ if self.in_flight.is_some() {
+ return Ok(None);
+ }
+ let Some(region) = self.regions.front() else {
+ return Ok(None);
+ };
+ let end = region
+ .end
+ .unwrap_or(self.ring.accepted())
+ .min(region.start + MAX_WINDOW_SAMPLES as u64);
+ let final_phrase = region.end == Some(end);
+ let at_limit = end - region.start == MAX_WINDOW_SAMPLES as u64;
+ if !final_phrase
+ && !at_limit
+ && (end - region.start < UPDATE_SAMPLES || end < self.last_decoded + UPDATE_SAMPLES)
+ {
+ return Ok(None);
+ }
+ self.next_id += 1;
+ self.in_flight = Some((self.next_id, region.start, end, final_phrase));
+ Ok(Some(DecodeWindow {
+ id: self.next_id,
+ start: region.start,
+ end,
+ final_phrase,
+ samples: self.ring.window(region.start, end)?,
+ }))
+ }
+
+ pub fn complete(
+ &mut self,
+ request: &DecodeWindow,
+ words: Vec<AlignedWord>,
+ ) -> Result<RecognitionUpdate> {
+ if self.in_flight != Some((request.id, request.start, request.end, request.final_phrase)) {
+ return Err(VoiceError::StaleEvent);
+ }
+ let mut current = words;
+ let mut previous_start = 0;
+ for word in &mut current {
+ if word.text.trim().is_empty()
+ || word.start < previous_start
+ || word.end < word.start
+ || word.end > request.end - request.start
+ {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ previous_start = word.start;
+ word.start += request.start;
+ word.end += request.start;
+ }
+ let skip = self.overlap_end(&current)?;
+ let remaining = &current[skip..];
+ let old_skip = self.overlap_end(&self.previous)?;
+ let mut observations = vec![1; current.len()];
+ let agreed_prefix = remaining
+ .iter()
+ .zip(&self.previous[old_skip..])
+ .take_while(|(a, b)| {
+ a.text == b.text && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE
+ })
+ .count();
+ for i in 0..agreed_prefix {
+ observations[skip + i] = self.observations[old_skip + i].saturating_add(1);
+ }
+ let mature = observations[skip..]
+ .iter()
+ .take_while(|count| **count >= REQUIRED_OBSERVATIONS)
+ .count()
+ .saturating_sub(RIGHT_CONTEXT_WORDS);
+ let agreed = remaining
+ .iter()
+ .take(mature)
+ .take_while(|word| word.end <= request.end.saturating_sub(LOOKBACK))
+ .count();
+ let commit_count = if request.final_phrase {
+ remaining.len()
+ } else {
+ agreed
+ };
+ if request.final_phrase && current.is_empty() {
+ return Err(VoiceError::IncompleteAudio);
+ }
+ let commit = if commit_count > 0 {
+ let end = if request.final_phrase {
+ request.end
+ } else {
+ remaining[commit_count - 1].end
+ };
+ if end <= self.covered {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ let text = join_words(&remaining[..commit_count]);
+ let start = self.covered.max(request.start);
+ self.covered = end;
+ self.anchor = Some(remaining[commit_count - 1].clone());
+ Some((start, end, text))
+ } else {
+ None
+ };
+ let hypothesis = join_words(&remaining[commit_count..]);
+ self.previous = current;
+ self.observations = observations;
+ self.last_decoded = request.end;
+ self.in_flight = None;
+ let mut retain_from = request.start;
+ if request.final_phrase {
+ self.covered = request.end;
+ self.regions.pop_front();
+ self.previous.clear();
+ self.observations.clear();
+ self.anchor = None;
+ self.last_decoded = 0;
+ retain_from = self.regions.front().map_or(request.end, |r| r.start);
+ retain_from = retain_from.min(request.end);
+ } else if request.end - request.start == MAX_WINDOW_SAMPLES as u64 {
+ if self.covered <= request.start + LOOKBACK {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ retain_from = self
+ .covered
+ .saturating_sub(LOOKBACK)
+ .min(self.anchor.as_ref().map_or(self.covered, |w| w.start))
+ .max(request.start);
+ if retain_from <= request.start {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ if let Some(region) = self.regions.front_mut() {
+ region.start = retain_from;
+ }
+ }
+ self.ring.acknowledge(
+ request.end.max(self.ring.processed()),
+ retain_from.max(self.ring.retained_from()),
+ )?;
+ self.release_silence()?;
+ Ok(RecognitionUpdate {
+ commit,
+ hypothesis,
+ covered_through: self.covered,
+ })
+ }
+
+ fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {
+ let Some(anchor) = &self.anchor else {
+ return Ok(0);
+ };
+ if words.is_empty() {
+ return Ok(0);
+ }
+ let anchor_key = alignment_key(&anchor.text);
+ let mut matches = words.iter().enumerate().filter_map(|(i, word)| {
+ (alignment_key(&word.text) == anchor_key
+ && word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE)
+ .then_some(i + 1)
+ });
+ match (matches.next(), matches.next()) {
+ (Some(end), None) => Ok(end),
+ (None, None) if words[0].start >= self.covered => Ok(0),
+ _ => Err(VoiceError::UnstableTranscript),
+ }
+ }
+
+ pub fn is_settled(&self) -> bool {
+ self.regions.is_empty() && self.in_flight.is_none()
+ }
+ #[cfg(test)]
+ fn covered(&self) -> u64 {
+ self.covered
+ }
+ #[cfg(test)]
+ fn retained_samples(&self) -> usize {
+ self.ring.retained_samples()
+ }
+}
+
+fn alignment_key(text: &str) -> String {
+ let lexical = text.trim().trim_matches([
+ '.', ',', '!', '?', ':', ';', '…', '。', ',', '!', '?', '؛', '،', '।', '॥', '¿', '¡',
+ '"', '“', '”',
+ ]);
+ if lexical.is_empty() {
+ text.trim().to_owned()
+ } else {
+ lexical.to_lowercase()
+ }
+}
+
+fn join_words(words: &[AlignedWord]) -> String {
+ let mut text = String::new();
+ for word in words {
+ text.push_str(boundary_separator(&text, &word.text));
+ text.push_str(&word.text);
+ }
+ text
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn aligned(text: &str, start: u64, end: u64) -> AlignedWord {
+ AlignedWord {
+ text: text.into(),
+ start,
+ end,
+ }
+ }
+
+ #[test]
+ fn provisional_revisions_need_repeated_right_context_before_commit() {
+ let mut rolling = RollingRecognizer::default();
+ for index in 0..5 {
+ rolling.feed(&vec![0.1; 8000], true).unwrap();
+ let request = rolling.next_window().unwrap().unwrap();
+ let first = if index < 2 { "Friday" } else { "Saturday" };
+ let update = rolling
+ .complete(
+ &request,
+ vec![
+ aligned(first, 0, 1000),
+ aligned("at", 1000, 2000),
+ aligned("noon", 2000, 3000),
+ ],
+ )
+ .unwrap();
+ if index < 4 {
+ assert!(update.commit.is_none());
+ assert_eq!(update.hypothesis, format!("{first} at noon"));
+ } else {
+ assert_eq!(update.commit.unwrap().2, "Saturday");
+ assert_eq!(update.hypothesis, "at noon");
+ }
+ }
+ }
+
+ #[test]
+ fn repeated_unfinished_tail_stays_revisable_until_endpoint() {
+ let mut rolling = RollingRecognizer::default();
+ for _ in 0..6 {
+ rolling.feed(&vec![0.1; 8000], true).unwrap();
+ let request = rolling.next_window().unwrap().unwrap();
+ let update = rolling
+ .complete(
+ &request,
+ vec![aligned("This", 0, 1000), aligned("one.", 1000, 2000)],
+ )
+ .unwrap();
+ assert!(update.commit.is_none());
+ }
+ rolling.finish();
+ let request = rolling.next_window().unwrap().unwrap();
+ let update = rolling
+ .complete(
+ &request,
+ vec![aligned("And", 0, 1000), aligned("so", 1000, 2000)],
+ )
+ .unwrap();
+ assert_eq!(update.commit.unwrap().2, "And so");
+ assert!(rolling.is_settled());
+ }
+
+ #[test]
+ fn short_vad_dropout_preserves_contiguous_audio_and_phrase() {
+ let mut rolling = RollingRecognizer::default();
+ rolling.feed(&vec![0.1; 8000], false).unwrap();
+ rolling.feed(&vec![0.2; 8000], true).unwrap();
+ rolling.feed(&vec![0.3; 16000], false).unwrap();
+ rolling.feed(&vec![0.4; 8000], true).unwrap();
+ let request = rolling.next_window().unwrap().unwrap();
+ assert_eq!((request.start, request.end), (0, 40000));
+ assert!(!request.final_phrase);
+ assert_eq!(&request.samples[16000..32000], &vec![0.3; 16000]);
+ assert_eq!(rolling.regions.len(), 1);
+ }
+
+ #[test]
+ fn overlap_ignores_boundary_punctuation_but_preserves_acoustic_identity() {
+ let rolling = RollingRecognizer {
+ anchor: Some(aligned("You.", 16000, 20000)),
+ covered: 20000,
+ ..RollingRecognizer::default()
+ };
+ assert_eq!(
+ rolling.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),
+ Ok(1)
+ );
+ assert_eq!(
+ rolling.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ assert_eq!(
+ rolling.overlap_end(&[aligned("you", 12000, 20000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ assert_eq!(
+ rolling.overlap_end(&[aligned("not", 16000, 20000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ assert_ne!(alignment_key("don't"), alignment_key("do"));
+ assert_ne!(alignment_key("कि"), alignment_key("क"));
+ assert_ne!(alignment_key("-5"), alignment_key("5"));
+ let rolling = RollingRecognizer {
+ anchor: Some(aligned("Café!", 16000, 20000)),
+ covered: 20000,
+ ..RollingRecognizer::default()
+ };
+ assert_eq!(rolling.overlap_end(&[aligned("café", 16000, 20000)]), Ok(1));
+ assert_eq!(
+ rolling.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ }
+
+ fn words(request: &DecodeWindow) -> Vec<AlignedWord> {
+ (0..request.end / 4000)
+ .filter_map(|i| {
+ let start = i * 4000;
+ (start >= request.start).then(|| AlignedWord {
+ text: "no".into(),
+ start: start - request.start,
+ end: (start + 3000).min(request.end) - request.start,
+ })
+ })
+ .collect()
+ }
+
+ #[test]
+ fn continuous_speech_advances_stable_text_without_losing_repetitions() {
+ let mut rolling = RollingRecognizer::default();
+ let mut committed = String::new();
+ for _ in 0..7200 {
+ rolling.feed(&vec![0.2; 4000], true).unwrap();
+ if let Some(request) = rolling.next_window().unwrap() {
+ assert!(request.samples.len() <= MAX_WINDOW_SAMPLES);
+ let update = rolling.complete(&request, words(&request)).unwrap();
+ if let Some((_, _, text)) = update.commit {
+ committed.push_str(&text);
+ committed.push(' ');
+ }
+ }
+ assert!(rolling.retained_samples() <= 30 * 16000);
+ }
+ assert!(committed.split_whitespace().count() > 7100);
+ rolling.finish();
+ while let Some(request) = rolling.next_window().unwrap() {
+ if let Some((_, _, text)) = rolling.complete(&request, words(&request)).unwrap().commit
+ {
+ committed.push_str(&text);
+ committed.push(' ');
+ }
+ }
+ assert!(rolling.is_settled());
+ assert_eq!(committed.split_whitespace().count(), 7200);
+ }
+
+ #[test]
+ fn final_short_speech_and_stale_results_are_handled() {
+ let mut rolling = RollingRecognizer::default();
+ rolling.feed(&vec![0.1; 3200], true).unwrap();
+ assert!(rolling.next_window().unwrap().is_none());
+ rolling.finish();
+ let request = rolling.next_window().unwrap().unwrap();
+ let update = rolling
+ .complete(
+ &request,
+ vec![AlignedWord {
+ text: "Ha".into(),
+ start: 0,
+ end: 3200,
+ }],
+ )
+ .unwrap();
+ assert_eq!(update.commit.unwrap().2, "Ha");
+ assert!(matches!(
+ rolling.complete(&request, Vec::new()),
+ Err(VoiceError::StaleEvent)
+ ));
+ }
+
+ #[test]
+ fn failed_alignment_keeps_audio_and_does_not_claim_completion() {
+ let mut rolling = RollingRecognizer::default();
+ rolling.feed(&vec![0.1; 8000], true).unwrap();
+ let request = rolling.next_window().unwrap().unwrap();
+ assert!(
+ rolling
+ .complete(
+ &request,
+ vec![AlignedWord {
+ text: "bad".into(),
+ start: 0,
+ end: 9000
+ }]
+ )
+ .is_err()
+ );
+ assert_eq!(rolling.retained_samples(), 8000);
+ assert_eq!(rolling.covered(), 0);
+ }
+}
diff --git a/rust/crates/ensu/src/voice/session.rs b/rust/crates/ensu/src/voice/session.rs
new file mode 100644
index 0000000000..613015d145
--- /dev/null
+++ b/rust/crates/ensu/src/voice/session.rs
@@ -0,0 +1,332 @@
+use super::{Result, VOICE_SAMPLE_RATE, VoiceError};
+
+const SILENCE_SAMPLES: u64 = 3 * VOICE_SAMPLE_RATE;
+const COUNTDOWN_SAMPLES: u64 = VOICE_SAMPLE_RATE * 3 / 5;
+const NO_SPEECH_SAMPLES: u64 = 15 * VOICE_SAMPLE_RATE;
+const SESSION_LIMIT_SAMPLES: u64 = 30 * 60 * VOICE_SAMPLE_RATE;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum VoicePhase {
+ Preparing,
+ Listening,
+ Countdown,
+ Finishing,
+ Ready,
+ Closed,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StopReason {
+ Done,
+ Silence,
+ Interruption,
+ Failure,
+ Limit,
+ NoSpeech,
+ Discard,
+}
+
+#[derive(Debug)]
+pub struct VoiceSession {
+ phase: VoicePhase,
+ accepted: u64,
+ recognized: u64,
+ waits_for_live_text: bool,
+ last_voice: Option<u64>,
+ stop_reason: Option<StopReason>,
+ worker_active: bool,
+ incomplete: bool,
+}
+
+impl Default for VoiceSession {
+ fn default() -> Self {
+ Self {
+ phase: VoicePhase::Preparing,
+ accepted: 0,
+ recognized: 0,
+ waits_for_live_text: true,
+ last_voice: None,
+ stop_reason: None,
+ worker_active: false,
+ incomplete: false,
+ }
+ }
+}
+
+impl VoiceSession {
+ pub fn recorded() -> Self {
+ Self {
+ waits_for_live_text: false,
+ ..Self::default()
+ }
+ }
+ pub fn phase(&self) -> VoicePhase {
+ self.phase
+ }
+ pub fn accepted_samples(&self) -> u64 {
+ self.accepted
+ }
+ pub fn worker_active(&self) -> bool {
+ self.worker_active
+ }
+ pub fn incomplete(&self) -> bool {
+ self.incomplete
+ }
+ pub fn stop_reason(&self) -> Option<StopReason> {
+ self.stop_reason
+ }
+
+ pub fn begin_capture(&mut self) -> Result<()> {
+ if self.phase != VoicePhase::Preparing {
+ return Err(VoiceError::InvalidState);
+ }
+ self.worker_active = true;
+ self.phase = VoicePhase::Listening;
+ Ok(())
+ }
+
+ pub fn defer_recognition(&mut self) {
+ self.waits_for_live_text = false;
+ self.update_countdown();
+ }
+
+ pub fn observe_audio(
+ &mut self,
+ start: u64,
+ end: u64,
+ voiced_end: Option<u64>,
+ ) -> Result<Option<StopReason>> {
+ if !matches!(self.phase, VoicePhase::Listening | VoicePhase::Countdown) {
+ return Err(VoiceError::InvalidState);
+ }
+ if start != self.accepted
+ || end <= start
+ || voiced_end.is_some_and(|v| v <= start || v > end)
+ {
+ return Err(VoiceError::InvalidAudio);
+ }
+ self.accepted = end;
+ if let Some(voice) = voiced_end {
+ self.last_voice = Some(voice);
+ self.phase = VoicePhase::Listening;
+ }
+ if end >= SESSION_LIMIT_SAMPLES {
+ return Ok(Some(StopReason::Limit));
+ }
+ if self.last_voice.is_none() && end >= NO_SPEECH_SAMPLES {
+ return Ok(Some(StopReason::NoSpeech));
+ }
+ self.update_countdown();
+ Ok(None)
+ }
+
+ pub fn observe_recognition(&mut self, has_text: bool) -> Result<()> {
+ if !self.worker_active || self.phase == VoicePhase::Closed {
+ return Err(VoiceError::InvalidState);
+ }
+ if has_text && self.waits_for_live_text {
+ self.waits_for_live_text = false;
+ self.update_countdown();
+ }
+ Ok(())
+ }
+
+ fn silence_deadline(&self) -> Option<u64> {
+ self.last_voice.map(|voice| {
+ let deadline = voice + SILENCE_SAMPLES;
+ if !self.waits_for_live_text {
+ deadline
+ } else {
+ deadline.max(NO_SPEECH_SAMPLES)
+ }
+ })
+ }
+ fn update_countdown(&mut self) {
+ if matches!(self.phase, VoicePhase::Listening | VoicePhase::Countdown) {
+ self.phase = if self.silence_deadline().is_some_and(|deadline| {
+ self.accepted >= deadline - (SILENCE_SAMPLES - COUNTDOWN_SAMPLES)
+ }) {
+ VoicePhase::Countdown
+ } else {
+ VoicePhase::Listening
+ };
+ }
+ }
+ pub fn silence_elapsed(&self) -> bool {
+ self.phase == VoicePhase::Countdown
+ && self
+ .silence_deadline()
+ .is_some_and(|deadline| self.accepted >= deadline)
+ }
+ pub fn stop(&mut self, reason: StopReason) -> Result<bool> {
+ if self.phase == VoicePhase::Closed {
+ return Ok(false);
+ }
+ if reason == StopReason::Discard {
+ self.stop_reason = Some(reason);
+ self.phase = VoicePhase::Closed;
+ return Ok(true);
+ }
+ if !self.worker_active {
+ return Err(VoiceError::InvalidState);
+ }
+ let first = self.stop_reason.is_none();
+ if first || matches!(reason, StopReason::Interruption | StopReason::Failure) {
+ self.stop_reason = Some(reason);
+ self.phase = VoicePhase::Finishing;
+ }
+ Ok(first)
+ }
+ pub fn recognize_through(&mut self, end: u64) -> Result<()> {
+ if !self.worker_active {
+ return Err(VoiceError::InvalidState);
+ }
+ if end < self.recognized || end > self.accepted {
+ return Err(VoiceError::InvalidAudio);
+ }
+ self.recognized = end;
+ Ok(())
+ }
+ pub fn observe_drained_audio(&mut self, start: u64, end: u64) -> Result<()> {
+ if !self.worker_active || self.phase == VoicePhase::Closed || self.stop_reason.is_none() {
+ return Err(VoiceError::InvalidState);
+ }
+ if start != self.accepted || end <= start {
+ return Err(VoiceError::InvalidAudio);
+ }
+ self.accepted = end;
+ Ok(())
+ }
+ pub fn worker_exited(&mut self, successful: bool) -> Result<()> {
+ if !self.worker_active {
+ return Err(VoiceError::InvalidState);
+ }
+ let reason = self.stop_reason.ok_or(VoiceError::InvalidState)?;
+ self.worker_active = false;
+ self.incomplete = !successful
+ || self.recognized < self.accepted
+ || matches!(reason, StopReason::Interruption | StopReason::Failure);
+ if self.phase != VoicePhase::Closed {
+ self.phase = VoicePhase::Ready;
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn speaking() -> VoiceSession {
+ let mut session = VoiceSession::default();
+ session.begin_capture().unwrap();
+ session.observe_audio(0, 16000, Some(16000)).unwrap();
+ session.observe_recognition(true).unwrap();
+ session
+ }
+
+ #[test]
+ fn silence_uses_the_same_audio_clock_after_live_or_deferred_recognition() {
+ for (mut session, deferred) in [
+ (speaking(), false),
+ (VoiceSession::recorded(), false),
+ (VoiceSession::default(), true),
+ ] {
+ if session.phase() == VoicePhase::Preparing {
+ session.begin_capture().unwrap();
+ session.observe_audio(0, 16000, Some(16000)).unwrap();
+ }
+ if deferred {
+ session.defer_recognition();
+ }
+ session.observe_audio(16000, 32000, None).unwrap();
+ assert!(!session.silence_elapsed());
+ session.observe_audio(32000, 64000, None).unwrap();
+ assert!(session.silence_elapsed());
+ assert!(session.stop(StopReason::Silence).unwrap());
+ assert!(!session.stop(StopReason::Done).unwrap());
+ assert!(session.worker_active());
+ assert_eq!(session.begin_capture(), Err(VoiceError::InvalidState));
+ session.recognize_through(64000).unwrap();
+ session.worker_exited(true).unwrap();
+ assert_eq!(session.phase(), VoicePhase::Ready);
+ assert!(!session.incomplete());
+ }
+ }
+
+ #[test]
+ fn live_grace_waits_for_text_or_fifteen_seconds() {
+ let mut session = VoiceSession::default();
+ session.begin_capture().unwrap();
+ session.observe_audio(0, 32000, Some(24000)).unwrap();
+ session.observe_recognition(false).unwrap();
+ session.observe_audio(32000, 80000, None).unwrap();
+ assert_eq!(session.phase(), VoicePhase::Listening);
+ session.observe_audio(80000, 13 * 16000, None).unwrap();
+ assert!(!session.silence_elapsed());
+ assert_eq!(session.silence_deadline(), Some(15 * 16000));
+ session.observe_audio(13 * 16000, 15 * 16000, None).unwrap();
+ assert!(session.silence_elapsed());
+ }
+
+ #[test]
+ fn recognition_and_later_speech_recompute_the_current_deadline() {
+ let mut session = VoiceSession::default();
+ session.begin_capture().unwrap();
+ session.observe_audio(0, 16000, Some(16000)).unwrap();
+ session.observe_audio(16000, 13 * 16000, None).unwrap();
+ assert!(!session.silence_elapsed());
+ session.observe_recognition(true).unwrap();
+ assert!(session.silence_elapsed());
+ session
+ .observe_audio(13 * 16000, 14 * 16000, Some(14 * 16000))
+ .unwrap();
+ assert!(!session.silence_elapsed());
+ assert_eq!(session.phase(), VoicePhase::Listening);
+ session.observe_audio(14 * 16000, 16 * 16000, None).unwrap();
+ assert!(!session.silence_elapsed());
+ session.observe_audio(16 * 16000, 17 * 16000, None).unwrap();
+ assert!(session.silence_elapsed());
+ }
+
+ #[test]
+ fn silence_and_duration_limits_apply_to_both_tiers() {
+ for mut session in [VoiceSession::default(), VoiceSession::recorded()] {
+ session.begin_capture().unwrap();
+ assert_eq!(
+ session.observe_audio(0, NO_SPEECH_SAMPLES, None).unwrap(),
+ Some(StopReason::NoSpeech)
+ );
+ assert!(!session.silence_elapsed());
+ assert_eq!(
+ session
+ .observe_audio(
+ NO_SPEECH_SAMPLES,
+ SESSION_LIMIT_SAMPLES,
+ Some(SESSION_LIMIT_SAMPLES)
+ )
+ .unwrap(),
+ Some(StopReason::Limit)
+ );
+ }
+ }
+
+ #[test]
+ fn interruption_retains_ownership_until_exit_and_cannot_resume() {
+ let mut session = speaking();
+ session.stop(StopReason::Done).unwrap();
+ session.stop(StopReason::Interruption).unwrap();
+ assert!(session.worker_active());
+ assert_eq!(session.phase(), VoicePhase::Finishing);
+ session.observe_drained_audio(16000, 32000).unwrap();
+ session.worker_exited(false).unwrap();
+ assert!(session.incomplete());
+ assert_eq!(session.phase(), VoicePhase::Ready);
+ assert_eq!(session.begin_capture(), Err(VoiceError::InvalidState));
+ let mut fresh = VoiceSession::default();
+ fresh.begin_capture().unwrap();
+ fresh.observe_audio(0, 16000, Some(16000)).unwrap();
+ fresh.observe_audio(16000, 80000, None).unwrap();
+ assert!(!fresh.silence_elapsed());
+ }
+}