Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Finalize mobile dictation recovery and controls

setalp7fbc4076c250
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 537b82b3f6..149067b9ff 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -92,6 +92,15 @@ class AppStore(
private var composerRevision = 0uL
private var emptyComposerId = UUID.randomUUID().toString()
+ private var composerSelectionStart = 0
+ private var composerSelectionEnd = 0
+ private var composerIsComposing = false
+
+ internal fun updateComposerSelection(start: Int, end: Int, isComposing: Boolean) {
+ composerSelectionStart = start
+ composerSelectionEnd = end
+ composerIsComposing = isComposing
+ }
private fun invalidateComposer() {
composerRevision++
@@ -138,14 +147,30 @@ class AppStore(
if (selectionStart !in 0..selectionEnd || selectionEnd > text.length) return
val characters = BreakIterator.getCharacterInstance(Locale.ROOT).apply { setText(text) }
if (!characters.isBoundary(selectionStart) || !characters.isBoundary(selectionEnd)) return
- voiceController.open(
- DictationInsertion(
- chat.currentSessionId ?: emptyComposerId,
- composerRevision,
- selectionStart.toUInt(),
- selectionEnd.toUInt(),
+ updateComposerSelection(selectionStart, selectionEnd, isComposing)
+ val destination = chat.currentSessionId ?: emptyComposerId
+ voiceController.open {
+ val current = _state.value.chat
+ val start = composerSelectionStart
+ val end = composerSelectionEnd
+ val boundaries =
+ BreakIterator.getCharacterInstance(Locale.ROOT).apply {
+ setText(current.messageText)
+ }
+ if (
+ destination != (current.currentSessionId ?: emptyComposerId) ||
+ current.isGenerating ||
+ current.isDownloading ||
+ current.editingMessageId != null ||
+ composerIsComposing ||
+ start !in 0..end ||
+ end > current.messageText.length ||
+ !boundaries.isBoundary(start) ||
+ !boundaries.isBoundary(end)
)
- )
+ null
+ else DictationInsertion(destination, composerRevision, start.toUInt(), end.toUInt())
+ }
}
fun dispose() = voiceController.dispose()
@@ -171,7 +196,10 @@ class AppStore(
modelSettingsActions.setAppForeground(active)
}
- fun setPickerPending(pending: Boolean) = modelSettingsActions.setPickerPending(pending)
+ fun setPickerPending(pending: Boolean) {
+ if (pending) voiceController.discard(keepDownload = true)
+ modelSettingsActions.setPickerPending(pending)
+ }
fun handleMemoryPressure() {
voiceController.interrupt(VoiceInputFailure.MemoryUnavailable)
@@ -246,27 +274,27 @@ class AppStore(
}
fun createNewSession(): String {
- voiceController.discard()
+ voiceController.discard(keepDownload = true)
invalidateComposer()
return chatActions.createNewSession()
}
fun startNewSessionDraft() {
- voiceController.discard()
+ voiceController.discard(keepDownload = true)
emptyComposerId = UUID.randomUUID().toString()
invalidateComposer()
chatActions.startNewSessionDraft()
}
fun selectSession(sessionId: String) {
- voiceController.discard()
+ voiceController.discard(keepDownload = true)
invalidateComposer()
chatActions.selectSession(sessionId)
}
fun deleteSession(sessionId: String) {
if (sessionId == _state.value.chat.currentSessionId) {
- voiceController.discard()
+ voiceController.discard(keepDownload = true)
invalidateComposer()
}
chatActions.deleteSession(sessionId)
@@ -286,7 +314,7 @@ class AppStore(
chatActions.updateBranchSelection(messageId, selectedIndex)
fun beginEditing(messageId: String) {
- voiceController.discard()
+ voiceController.discard(keepDownload = true)
invalidateComposer()
chatActions.beginEditing(messageId)
}
@@ -335,10 +363,14 @@ class AppStore(
modelSettingsActions.cancelModelDownload()
}
- fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)
+ fun retryAssistantMessage(messageId: String) {
+ voiceController.interrupt()
+ chatActions.retryAssistantMessage(messageId)
+ }
fun sendMessage() {
- if (voiceController.active) return
+ if (voiceController.blocksComposer) return
+ voiceController.discard(keepDownload = true)
invalidateComposer()
chatActions.sendMessage()
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index b306bf813e..f03aae6ee8 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -128,6 +128,7 @@ internal fun HomeNavigation(
chatState = appState.chat,
isDrawerOpen = drawerState.currentValue == DrawerValue.Open,
onMessageChange = store::updateMessageText,
+ onComposerSelection = store::updateComposerSelection,
onSend = store::sendMessage,
onStop = store::stopGeneration,
onCancelDownload = store::cancelDownload,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
index 9c3938b402..8c7f62065f 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatInputBar.kt
@@ -39,6 +39,7 @@ import androidx.compose.ui.focus.FocusRequester
import androidx.compose.ui.focus.focusRequester
import androidx.compose.ui.graphics.SolidColor
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
+import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.text.TextRange
import androidx.compose.ui.text.input.KeyboardCapitalization
@@ -69,6 +70,7 @@ internal fun MessageInput(
sourcesError: String?,
onRetrySources: () -> Unit,
onMessageChange: (String) -> Unit,
+ onComposerSelection: (Int, Int, Boolean) -> Unit,
onSend: () -> Unit,
onStop: () -> Unit,
onAttachmentSelected: (AttachmentType) -> Unit,
@@ -80,6 +82,7 @@ internal fun MessageInput(
focusRequestId: Int,
) {
val haptic = rememberHaptics()
+ val focusManager = LocalFocusManager.current
val placeholder = "Ask Ensu"
val composerIconSize = 18.dp
@@ -93,7 +96,7 @@ internal fun MessageInput(
)
}
- LaunchedEffect(messageText, voiceCaret, voiceController.active) {
+ LaunchedEffect(messageText, voiceCaret, voiceController.blocksComposer) {
if (messageText != fieldValue.text || voiceCaret != null) {
fieldValue =
TextFieldValue(
@@ -104,6 +107,13 @@ internal fun MessageInput(
),
)
}
+ if (!voiceController.blocksComposer) {
+ onComposerSelection(
+ fieldValue.selection.min,
+ fieldValue.selection.max,
+ fieldValue.composition != null,
+ )
+ }
}
LaunchedEffect(focusRequestId) {
@@ -218,7 +228,7 @@ internal fun MessageInput(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
) {
- if (supportsImageAttachments && !voiceController.active) {
+ if (supportsImageAttachments && !voiceController.blocksComposer) {
IconButton(
onClick = {
haptic.perform(HapticFeedbackType.TextHandleMove)
@@ -246,14 +256,21 @@ internal fun MessageInput(
value = fieldValue,
onValueChange = { newValue ->
fieldValue = newValue
- if (!voiceController.active) onMessageChange(newValue.text)
+ if (!voiceController.blocksComposer) {
+ onMessageChange(newValue.text)
+ onComposerSelection(
+ newValue.selection.min,
+ newValue.selection.max,
+ newValue.composition != null,
+ )
+ }
},
modifier = Modifier.weight(1f).focusRequester(focusRequester),
textStyle = EnsuTypography.message.copy(color = EnsuColor.textPrimary()),
visualTransformation = LongMessageVisualTransformation,
minLines = 1,
- maxLines = if (voiceController.active) 8 else 5,
- readOnly = voiceController.active,
+ maxLines = if (voiceController.blocksComposer) 8 else 5,
+ readOnly = voiceController.blocksComposer,
keyboardOptions =
KeyboardOptions(capitalization = KeyboardCapitalization.Sentences),
cursorBrush = SolidColor(EnsuColor.accent()),
@@ -276,9 +293,17 @@ internal fun MessageInput(
if (editingMessage == null) {
VoiceInputButton(
controller = voiceController,
- enabled = !isGenerating && !isDownloading && fieldValue.composition == null,
+ enabled = !isGenerating && !isDownloading,
onOpen = {
haptic.perform(HapticFeedbackType.TextHandleMove)
+ focusManager.clearFocus(force = true)
+ fieldValue = fieldValue.copy(composition = null)
+ onMessageChange(fieldValue.text)
+ onComposerSelection(
+ fieldValue.selection.min,
+ fieldValue.selection.max,
+ false,
+ )
onVoiceInput(fieldValue)
},
)
@@ -287,7 +312,7 @@ internal fun MessageInput(
val canSend =
sourcesReady &&
(messageText.isNotBlank() || attachments.isNotEmpty()) &&
- !voiceController.active &&
+ !voiceController.blocksComposer &&
!hasUnsupportedImages
IconButton(
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 6b0d17dca9..b83f6be40e 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -947,9 +947,6 @@ internal class ChatStoreActions(
systemPrompt = normalSystemPrompt,
)
- if (normalHistorySelection.currentInputTooLarge) {
- throw ConversationException.CurrentInputTooLarge()
- }
if (
normalHistorySelection.wasTrimmed &&
overflowBypassMessageId != userMessage.id
@@ -1089,18 +1086,6 @@ internal class ChatStoreActions(
throw err
} catch (_: LlmException.Cancelled) {
interrupted = true
- } catch (_: ConversationException.CurrentInputTooLarge) {
- if (isActive() && !stopRequested) {
- state.update {
- it.copy(
- chat =
- it.chat.copy(
- transientAssistantError = CURRENT_INPUT_TOO_LARGE_MESSAGE,
- transientAssistantParentId = userMessage.id,
- )
- )
- }
- }
} catch (err: ModelMemoryUnavailable) {
if (isActive() && !stopRequested) {
state.update {
@@ -1861,7 +1846,6 @@ internal class ChatStoreActions(
val inputTokens: Int,
val inputBudget: Int,
val wasTrimmed: Boolean,
- val currentInputTooLarge: Boolean,
)
private data class GenerationLimits(
@@ -1933,7 +1917,6 @@ internal class ChatStoreActions(
inputTokens,
inputBudget,
inputTokens > inputBudget,
- systemTokens + promptTokens > inputBudget,
)
}
@@ -1964,13 +1947,7 @@ internal class ChatStoreActions(
)
}
- return HistorySelection(
- selected,
- inputTokens,
- inputBudget,
- inputTokens > inputBudget,
- false,
- )
+ return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)
}
private fun resolveGenerationLimits(contextLength: Int): GenerationLimits {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index 9bce45e391..5f929cef14 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -50,6 +50,7 @@ internal fun ChatView(
onOpenVoiceInput: (androidx.compose.ui.text.input.TextFieldValue) -> Unit,
isDrawerOpen: Boolean,
onMessageChange: (String) -> Unit,
+ onComposerSelection: (Int, Int, Boolean) -> Unit,
onSend: () -> Unit,
onStop: () -> Unit,
onCancelDownload: () -> Unit,
@@ -235,7 +236,7 @@ internal fun ChatView(
inputBarHeightDp = with(density) { coords.size.height.toDp() }
}
)
- } else if (chatState.isModelStateKnown) {
+ } else if (chatState.isModelStateKnown && !showDownloadOnboarding) {
val voicePreview = voiceController.composerPreview(chatState.messageText)
MessageInput(
modifier =
@@ -256,6 +257,7 @@ internal fun ChatView(
sourcesError = chatState.sourceSelectionError,
onRetrySources = onRetrySources,
onMessageChange = onMessageChange,
+ onComposerSelection = onComposerSelection,
onSend = {
focusManager.clearFocus()
onSend()
@@ -265,10 +267,7 @@ internal fun ChatView(
onRemoveAttachment = onRemoveAttachment,
onCancelEdit = onCancelEdit,
voiceController = voiceController,
- onVoiceInput = { value ->
- onOpenVoiceInput(value)
- focusManager.clearFocus()
- },
+ onVoiceInput = onOpenVoiceInput,
voiceCaret = voicePreview?.caretUtf16?.toInt() ?: voiceController.composerCaret,
focusRequestId = focusRequestId,
)
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index 55a0c40e4e..352571fc92 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -57,6 +57,7 @@ internal enum class VoiceInputFailure(val message: String) {
MemoryUnavailable("Not enough free memory. Close other apps and try again."),
MicrophoneUnavailable("Could not start the microphone. Tap the microphone to try again."),
Interrupted("Voice input was interrupted. Review the available text and try again."),
+ Incomplete("Some words may be missing. Review the text."),
ProcessingFailed("Could not transcribe voice input. Review the available text and try again."),
}
@@ -84,7 +85,7 @@ internal sealed interface VoiceInputState {
data object Recording : VoiceInputState
- data class Downloading(val percent: Int?) : VoiceInputState
+ data class Downloading(val percent: Int?, val queued: Boolean = true) : VoiceInputState
data object Transcribing : VoiceInputState
@@ -130,8 +131,10 @@ internal class VoiceTranscriptionController(
private var capture: LiveMicrophone? = null
private var work: Job? = null
private var finishingNoticeJob: Job? = null
+ private var transientErrorJob: Job? = null
private var attemptFinished = false
private var insertion: DictationInsertion? = null
+ private var insertionFactory: (() -> DictationInsertion?)? = null
private var foreground = true
private var generation = 0L
val sessionVersion: Long
@@ -166,13 +169,17 @@ internal class VoiceTranscriptionController(
val canOpen: Boolean
get() = !active && !hasActiveWork
- fun open(insertion: DictationInsertion) {
+ val blocksComposer: Boolean
+ get() = active && state != VoiceInputState.Idle && state !is VoiceInputState.Downloading
+
+ fun open(insertionFactory: () -> DictationInsertion?) {
if (!canOpen) return
+ transientErrorJob?.cancel()
generation++
initialRecordPending = true
attemptFinished = false
isFinishingSlow = false
- this.insertion = insertion
+ this.insertionFactory = insertionFactory
session = DictationSession(live = usesLiveDictation)
snapshot = session?.snapshot()
failure = null
@@ -183,7 +190,7 @@ internal class VoiceTranscriptionController(
fun composerPreview(original: String): io.ente.ensu.bindings.DictationInsertionResult? {
val value = snapshot ?: return null
- if (!active) return null
+ if (!blocksComposer) return null
return runCatching { insertion?.preview(original, value.text + value.hypothesis) }
.getOrNull()
}
@@ -211,14 +218,17 @@ internal class VoiceTranscriptionController(
startOnForeground = false
record()
} else session?.let { refresh(it, generation) }
- } else if (!awaitingPermission && !startOnForeground) interrupt()
+ } else if (
+ !awaitingPermission && !startOnForeground && state !is VoiceInputState.Downloading
+ )
+ interrupt()
}
fun record() {
val owner = session ?: return
if (!active || !foreground || hasActiveWork || attemptFinished) return
failure = null
- state = VoiceInputState.Preparing
+ state = VoiceInputState.Downloading(null)
val token = generation
hasActiveWork = true
work =
@@ -232,16 +242,30 @@ internal class VoiceTranscriptionController(
modelAssets.all { assetStore.isDownloaded(it) }
}
) {
- state = VoiceInputState.Downloading(null)
assetStore.download(modelAssets) { progress ->
scope.launch {
if (token == generation && state is VoiceInputState.Downloading)
state =
- VoiceInputState.Downloading(progress.percentage.toInt())
+ VoiceInputState.Downloading(
+ progress.totalBytes?.let {
+ progress.percentage.toInt()
+ },
+ progress.queued,
+ )
}
}
}
- if (token != generation || !foreground) return@launch
+ if (token != generation) return@launch
+ if (!foreground) {
+ discard()
+ return@launch
+ }
+ insertion = insertionFactory?.invoke()
+ insertionFactory = null
+ if (insertion == null) {
+ discard()
+ return@launch
+ }
state = VoiceInputState.Preparing
val activeStream = DictationStream(owner, 16000u, 1u)
stream = activeStream
@@ -370,6 +394,8 @@ internal class VoiceTranscriptionController(
val completedState =
when {
added && !value.incomplete -> VoiceInputState.Idle
+ added && failure == VoiceInputFailure.ProcessingFailed ->
+ VoiceInputState.Error(VoiceInputFailure.Incomplete)
text.isBlank() -> VoiceInputState.Error(failure ?: VoiceInputFailure.NoSpeech)
else -> VoiceInputState.Error(failure ?: VoiceInputFailure.Interrupted)
}
@@ -378,8 +404,15 @@ internal class VoiceTranscriptionController(
snapshot = null
session = null
insertion = null
+ insertionFactory = null
audioLevels = List(24) { 0f }
state = completedState
+ if (completedState == VoiceInputState.Error(VoiceInputFailure.NoSpeech)) {
+ transientErrorJob = scope.launch {
+ delay(10_000)
+ if (token == generation && state == completedState) state = VoiceInputState.Idle
+ }
+ }
}
private fun watchFinalization(
@@ -426,6 +459,10 @@ internal class VoiceTranscriptionController(
fun interrupt(reason: VoiceInputFailure = VoiceInputFailure.Interrupted) {
if (!active || attemptFinished) return
+ if (state is VoiceInputState.Downloading) {
+ insertionFactory = null
+ return
+ }
if (!hasActiveWork && stream == null) {
discard()
return
@@ -439,10 +476,16 @@ internal class VoiceTranscriptionController(
isFinishingSlow = false
}
- fun discard() {
+ fun discard(keepDownload: Boolean = false) {
+ if (keepDownload && state is VoiceInputState.Downloading && hasActiveWork) {
+ insertionFactory = null
+ return
+ }
awaitingPermission = false
startOnForeground = false
generation++
+ transientErrorJob?.cancel()
+ transientErrorJob = null
finishingNoticeJob?.cancel()
finishingNoticeJob = null
capture?.stop()
@@ -454,6 +497,7 @@ internal class VoiceTranscriptionController(
snapshot = null
session = null
insertion = null
+ insertionFactory = null
stream = null
capture = null
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
index dd79663204..f7e40398b3 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
@@ -38,6 +38,7 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.geometry.CornerRadius
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.hapticfeedback.HapticFeedbackType
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.semantics.LiveRegionMode
@@ -51,6 +52,7 @@ import io.ente.ensu.designsystem.EnsuColor
import io.ente.ensu.designsystem.EnsuSpacing
import io.ente.ensu.designsystem.EnsuTypography
import io.ente.ensu.designsystem.HugeIcons
+import io.ente.ensu.platform.rememberHaptics
@Composable
internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
@@ -60,7 +62,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
val context = LocalContext.current
val label =
when (state) {
- is VoiceInputState.Downloading -> "Downloading"
+ is VoiceInputState.Downloading -> if (state.queued) "Download queued" else "Downloading"
VoiceInputState.Preparing,
VoiceInputState.Idle -> "Preparing"
VoiceInputState.Transcribing ->
@@ -98,7 +100,7 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))
else Spacer(Modifier.weight(1f))
if (controller.active && !state.isRecording) {
- TextButton(onClick = controller::discard) { Text("Cancel") }
+ TextButton(onClick = { controller.discard() }) { Text("Cancel") }
}
}
}
@@ -110,6 +112,7 @@ internal fun VoiceInputButton(
onOpen: () -> Unit,
) {
val context = LocalContext.current
+ val haptic = rememberHaptics()
var permissionRequest by rememberSaveable { mutableStateOf<Long?>(null) }
val permission =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {
@@ -140,6 +143,7 @@ internal fun VoiceInputButton(
val canPress = if (controller.active) capturing else enabled && controller.canOpen
val description =
when {
+ downloading?.queued == true -> "Voice model download queued"
downloading != null -> "Downloading voice model"
preparing -> "Preparing voice input"
transcribing -> "Transcribing"
@@ -149,17 +153,20 @@ internal fun VoiceInputButton(
IconButton(
onClick = {
when {
- capturing -> controller.stop()
+ capturing -> {
+ haptic.perform(HapticFeedbackType.LongPress)
+ controller.stop()
+ }
else -> onOpen()
}
},
enabled = canPress,
- modifier = Modifier.size(44.dp).semantics { contentDescription = description },
+ modifier = Modifier.size(36.dp).semantics { contentDescription = description },
) {
when {
downloading != null || preparing || transcribing ->
Box(Modifier.size(28.dp), contentAlignment = Alignment.Center) {
- if (downloading?.percent != null)
+ if (downloading?.percent != null && !downloading.queued)
CircularProgressIndicator(
progress = { downloading.percent.coerceIn(0, 100) / 100f },
modifier = Modifier.size(28.dp),
@@ -187,7 +194,7 @@ internal fun VoiceInputButton(
tint = EnsuColor.backgroundBase(),
)
}
- else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(20.dp))
+ else -> Icon(painterResource(HugeIcons.Mic02Icon), null, Modifier.size(18.dp))
}
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swift
index c4ada28a33..be96b716a0 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift
@@ -364,7 +364,7 @@ struct ChatView: View {
}
.padding(.bottom, keyboard.isVisible ? keyboard.height : 0)
.zIndex(1)
- } else {
+ } else if viewModel.isModelDownloaded {
let shouldAutoFocus = shouldAutoFocusInput
MessageInputView(
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index fe2100e4fd..97325b6174 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -534,21 +534,9 @@ final class ChatViewModel: ObservableObject {
guard !isGenerating, !isDownloading, !isSendPending, editingMessageId == nil,
!composerIsComposing, !voiceTranscriber.active, !voiceTranscriber.hasActiveTasks
else { return }
- let range = composerSelection
- let text = draftText as NSString
- guard range.location >= 0, range.length >= 0, range.location <= text.length,
- range.length <= text.length - range.location,
- [range.location, NSMaxRange(range)].allSatisfy({
- $0 == text.length || text.rangeOfComposedCharacterSequence(at: $0).location == $0
- })
- else { return }
sessionSummaryTask?.cancel()
cancelChatWarmup(releaseLoadedModel: false)
let destination = (currentSessionId ?? emptyComposerId).uuidString
- let revision = composerRevision
- let insertion = DictationInsertion(
- destination: destination, revision: revision,
- startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))
voiceTranscriber.onInsert = { [weak self] insertion, text in
guard let self, self.appForeground else { return false }
do {
@@ -563,11 +551,29 @@ final class ChatViewModel: ObservableObject {
return true
} catch { return false }
}
- voiceTranscriber.open(insertion: insertion)
+ voiceTranscriber.open { [weak self] in
+ guard let self, self.appForeground, self.chatActive,
+ !self.isGenerating, !self.isDownloading, !self.isSendPending,
+ self.editingMessageId == nil, !self.composerIsComposing,
+ (self.currentSessionId ?? self.emptyComposerId).uuidString == destination
+ else { return nil }
+ let range = self.composerSelection
+ let text = self.draftText as NSString
+ guard range.location >= 0, range.length >= 0, range.location <= text.length,
+ range.length <= text.length - range.location,
+ [range.location, NSMaxRange(range)].allSatisfy({
+ $0 == text.length
+ || text.rangeOfComposedCharacterSequence(at: $0).location == $0
+ })
+ else { return nil }
+ return DictationInsertion(
+ destination: destination, revision: self.composerRevision,
+ startUtf16: UInt32(range.location), endUtf16: UInt32(NSMaxRange(range)))
+ }
}
func cancelVoiceInput() {
- voiceTranscriber.cancel()
+ voiceTranscriber.releaseComposer()
emptyComposerId = UUID()
voiceInsertionCaret = nil
}
@@ -715,7 +721,7 @@ final class ChatViewModel: ObservableObject {
}
func sendDraft() {
- guard !voiceTranscriber.active else { return }
+ guard !voiceTranscriber.blocksComposer else { return }
guard !isGenerating && !isDownloading && !isSendPending else { return }
guard !isChatUnsupported else {
showUnsupportedDeviceDialog = true
@@ -734,6 +740,7 @@ final class ChatViewModel: ObservableObject {
generationErrorMessage = ModelImagesUnsupported().localizedDescription
return
}
+ voiceTranscriber.releaseComposer()
let sourceSnapshot = currentSourceSelection()
let sendId = UUID()
pendingSendId = sendId
@@ -1143,6 +1150,7 @@ final class ChatViewModel: ObservableObject {
userNode = messageStore[sessionId]?.first(where: { $0.id == parentId })
}
guard let userNode else { return }
+ voiceTranscriber.interrupt()
let sourceSnapshot = currentSourceSelection()
cancelPendingSend()
priorGeneration?.cancel()
@@ -1595,9 +1603,6 @@ final class ChatViewModel: ObservableObject {
sessionId: parent.sessionId, promptText: prompt.text,
promptImageCount: prompt.imageFiles.count, currentMessageId: parent.id,
limits: limits, systemPrompt: systemPrompt)
- if normalHistory.currentInputTooLarge {
- throw ConversationError.CurrentInputTooLarge
- }
if normalHistory.wasTrimmed && overflowBypassMessageId != parent.id {
return .overflow(
inputTokens: normalHistory.inputTokens,
@@ -2503,7 +2508,6 @@ final class ChatViewModel: ObservableObject {
let inputTokens: Int
let inputBudget: Int
let wasTrimmed: Bool
- let currentInputTooLarge: Bool
}
private struct GenerationLimits {
@@ -2572,8 +2576,7 @@ final class ChatViewModel: ObservableObject {
if remaining <= 0 || historyMessages.isEmpty {
return HistorySelection(
messages: [], inputTokens: inputTokens, inputBudget: inputBudget,
- wasTrimmed: inputTokens > inputBudget,
- currentInputTooLarge: systemTokens + promptTokens > inputBudget)
+ wasTrimmed: inputTokens > inputBudget)
}
let quantum = max(1, inputBudget / 4)
@@ -2603,7 +2606,7 @@ final class ChatViewModel: ObservableObject {
return HistorySelection(
messages: selected, inputTokens: inputTokens, inputBudget: inputBudget,
- wasTrimmed: inputTokens > inputBudget, currentInputTooLarge: false)
+ wasTrimmed: inputTokens > inputBudget)
}
private func resolveMaxOutputTokens(contextLength: Int) -> Int {
diff --git a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
index f9bf3bd0d8..863aa72427 100644
--- a/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
+++ b/apple/apps/ensu/Ensu/Chat/MessageInputView.swift
@@ -41,7 +41,7 @@ struct MessageInputView: View {
let hasContent =
!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !attachments.isEmpty
return hasContent && !isDownloading && !isSendPending
- && !voiceService.active
+ && !voiceService.blocksComposer
}
private var isImageAttachmentLimitReached: Bool {
@@ -102,11 +102,12 @@ struct MessageInputView: View {
.padding(.horizontal, EnsuSpacing.pageHorizontal)
HStack(alignment: .bottom, spacing: EnsuSpacing.xs) {
- if supportsImageAttachments && !voiceService.active {
+ if supportsImageAttachments && !voiceService.blocksComposer {
ActionButton(
icon: "ImagePlusIcon", tooltip: "Add photos",
color: EnsuColor.textPrimary, size: 32
) {
+ voiceService.releaseComposer()
dismissInput()
showPhotos = true
}
@@ -136,7 +137,7 @@ struct MessageInputView: View {
MessageTextView(
text: Binding(
get: { composerText },
- set: { if !voiceService.active { text = $0 } }
+ set: { if !voiceService.blocksComposer { text = $0 } }
),
isFocused: $isFocused,
moveCursorToEndToken: moveCursorToEndToken,
@@ -151,8 +152,8 @@ struct MessageInputView: View {
insertionCaret: voicePreview.map { Int($0.caretUtf16) }
?? voiceInsertionCaret,
onSelectionChange: onComposerSelection,
- readOnly: voiceService.active,
- maximumLines: voiceService.active ? 8 : 5
+ readOnly: voiceService.blocksComposer,
+ maximumLines: voiceService.blocksComposer ? 8 : 5
)
.padding(.vertical, 6)
.overlay(alignment: .topLeading) {
@@ -182,6 +183,7 @@ struct MessageInputView: View {
onDismissKeyboard()
onVoiceInput()
}
+ .padding(.leading, EnsuSpacing.xs)
}
Button {
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
index 021cfc7f05..68f67b22fa 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
@@ -7,7 +7,7 @@ struct VoiceInputControls: View {
private var status: String {
switch service.state {
- case .downloading: return "Downloading"
+ case .downloading(_, let queued): return queued ? "Download queued" : "Downloading"
case .preparing: return "Preparing"
case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"
case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"
@@ -91,7 +91,8 @@ struct VoiceInputButton: View {
private var label: String {
switch service.state {
- case .downloading: return "Downloading voice model"
+ case .downloading(_, let queued):
+ return queued ? "Voice model download queued" : "Downloading voice model"
case .preparing: return "Preparing voice input"
case .transcribing: return "Transcribing"
case .recording: return "Stop dictation"
@@ -101,12 +102,17 @@ struct VoiceInputButton: View {
var body: some View {
Button {
- if service.state.isRecording { service.stop() } else { onOpen() }
+ if service.state.isRecording {
+ hapticWarning()
+ service.stop()
+ } else {
+ onOpen()
+ }
} label: {
ZStack {
switch service.state {
- case .downloading(let percent):
- if let percent {
+ case .downloading(let percent, let queued):
+ if let percent, !queued {
Circle().stroke(EnsuColor.textMuted.opacity(0.2), lineWidth: 2)
Circle().trim(from: 0, to: CGFloat(min(100, max(0, percent))) / 100)
.stroke(
@@ -125,11 +131,11 @@ struct VoiceInputButton: View {
RoundedRectangle(cornerRadius: 2).fill(EnsuColor.backgroundBase)
.frame(width: 12, height: 12)
case .idle, .error:
- Image("Mic02Icon").resizable().scaledToFit().frame(width: 20, height: 20)
+ Image("Mic02Icon").resizable().scaledToFit().frame(width: 16, height: 16)
}
}
.frame(width: 28, height: 28)
- .frame(width: 44, height: 44)
+ .frame(width: 32, height: 32)
.contentShape(Rectangle())
}
.foregroundStyle(canPress ? EnsuColor.textPrimary : EnsuColor.textMuted)
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 961975aa3d..6526995cce 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -5,7 +5,7 @@ import Combine
enum VoiceInputFailure: Equatable {
case permission, microphone, noSpeech, interrupted, captureInterrupted
case modelUnavailable, memoryUnavailable, storageFull, download, fileAccess, transcription
- case insertion
+ case insertion, incomplete
var message: String {
switch self {
@@ -31,6 +31,8 @@ enum VoiceInputFailure: Equatable {
return "Could not access voice model files. Try again."
case .transcription:
return "Could not transcribe voice input. Review the available text or try again."
+ case .incomplete:
+ return "Some words may be missing. Review the text."
case .insertion:
return "The draft changed before voice input finished. Tap the microphone to try again."
}
@@ -62,7 +64,7 @@ enum VoiceInputState: Equatable {
case idle
case preparing
case recording
- case downloading(percent: Int?)
+ case downloading(percent: Int?, queued: Bool)
case transcribing
case error(VoiceInputFailure)
@@ -87,7 +89,19 @@ final class VoiceTranscriptionService: ObservableObject {
@Published private(set) var snapshot: DictationSnapshot?
@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)
@Published private(set) var notice: VoiceInputFailure?
- @Published private(set) var state: VoiceInputState = .idle
+ @Published private(set) var state: VoiceInputState = .idle {
+ didSet {
+ transientErrorTask?.cancel()
+ transientErrorTask = nil
+ if state == .error(.noSpeech) {
+ transientErrorTask = Task { @MainActor [weak self] in
+ do { try await Task.sleep(for: .seconds(10)) } catch { return }
+ guard let self, self.state == .error(.noSpeech) else { return }
+ self.state = .idle
+ }
+ }
+ }
+ }
@Published private(set) var isFinishingSlow = false
var onActivityChanged: (@MainActor () -> Void)?
var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?
@@ -101,8 +115,10 @@ final class VoiceTranscriptionService: ObservableObject {
private var recorder: LivePcmRecorder?
private var task: Task<Void, Never>?
private var finishingNoticeTask: Task<Void, Never>?
+ private var transientErrorTask: Task<Void, Never>?
private var attemptFinished = false
private var insertion: DictationInsertion?
+ private var makeInsertion: (@MainActor () -> DictationInsertion?)?
private var generation = UUID()
private var foreground = true
private var awaitingPermission = false
@@ -155,10 +171,18 @@ final class VoiceTranscriptionService: ObservableObject {
}
}
- func open(insertion: DictationInsertion) {
+ var blocksComposer: Bool {
+ guard active else { return false }
+ switch state {
+ case .idle, .downloading: return false
+ default: return true
+ }
+ }
+
+ func open(makeInsertion: @escaping @MainActor () -> DictationInsertion?) {
guard !active, !hasActiveTasks else { return }
generation = UUID()
- self.insertion = insertion
+ self.makeInsertion = makeInsertion
attemptFinished = false
isFinishingSlow = false
session = DictationSession(live: usesLiveDictation)
@@ -179,6 +203,7 @@ final class VoiceTranscriptionService: ObservableObject {
refresh(owner, token: generation)
}
} else if !awaitingPermission, !startOnForeground {
+ if case .downloading = state { return }
interrupt()
}
}
@@ -221,7 +246,7 @@ final class VoiceTranscriptionService: ObservableObject {
}
hasActiveTasks = true
notice = nil
- state = .preparing
+ state = .downloading(percent: nil, queued: true)
task = Task { [weak self] in
guard let self else { return }
defer {
@@ -238,19 +263,30 @@ final class VoiceTranscriptionService: ObservableObject {
}
do {
if !self.modelAssets.allSatisfy({ self.assetStore.isDownloaded($0) }) {
- self.state = .downloading(percent: nil)
try await self.assetStore.download(assets: self.modelAssets) {
[weak self] progress in
Task { @MainActor [weak self] in
guard let self, self.generation == token,
case .downloading = self.state
else { return }
- self.state = .downloading(percent: Int(progress.percentage))
+ self.state = .downloading(
+ percent: progress.totalBytes.map { _ in Int(progress.percentage) },
+ queued: progress.queued)
}
}
}
try Task.checkCancellation()
- guard self.generation == token, self.foreground else { return }
+ guard self.generation == token else { return }
+ guard self.foreground else {
+ self.cancel()
+ return
+ }
+ guard let insertion = self.makeInsertion?() else {
+ self.cancel()
+ return
+ }
+ self.insertion = insertion
+ self.makeInsertion = nil
self.state = .preparing
let recorder: LivePcmRecorder
do {
@@ -345,7 +381,9 @@ final class VoiceTranscriptionService: ObservableObject {
let added = hasText && insertion.map { onInsert?($0, text) == true } == true
let completedState: VoiceInputState
if added {
- completedState = value.incomplete ? .error(notice ?? .interrupted) : .idle
+ let failure: VoiceInputFailure =
+ notice == .transcription ? .incomplete : notice ?? .interrupted
+ completedState = value.incomplete ? .error(failure) : .idle
} else {
completedState = .error(hasText ? .insertion : notice ?? .noSpeech)
}
@@ -354,6 +392,7 @@ final class VoiceTranscriptionService: ObservableObject {
session = nil
snapshot = nil
insertion = nil
+ makeInsertion = nil
onInsert = nil
audioLevels = Array(repeating: 0, count: 24)
state = completedState
@@ -387,6 +426,10 @@ final class VoiceTranscriptionService: ObservableObject {
func interrupt(reason: VoiceInputFailure = .interrupted) {
guard active, !attemptFinished else { return }
+ if case .downloading = state {
+ releaseComposer()
+ return
+ }
if !hasActiveTasks, stream == nil {
cancel()
return
@@ -400,6 +443,15 @@ final class VoiceTranscriptionService: ObservableObject {
state = .transcribing
}
+ func releaseComposer() {
+ if case .downloading = state {
+ makeInsertion = nil
+ onInsert = nil
+ } else {
+ cancel()
+ }
+ }
+
func cancel() {
awaitingPermission = false
startOnForeground = false
@@ -415,6 +467,7 @@ final class VoiceTranscriptionService: ObservableObject {
snapshot = nil
session = nil
insertion = nil
+ makeInsertion = nil
stream = nil
recorder = nil
onInsert = nil
diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rs
index c44343f6ac..3f6dfc3518 100644
--- a/rust/bindings/uniffi/ensu/src/assets.rs
+++ b/rust/bindings/uniffi/ensu/src/assets.rs
@@ -90,6 +90,7 @@ pub fn knowledge_embedding_model_asset() -> Arc<Asset> {
#[derive(Debug, Clone, uniffi::Record)]
pub struct AssetDownloadProgress {
pub label: String,
+ pub queued: bool,
pub downloaded_bytes: i64,
pub total_bytes: Option<i64>,
pub percentage: f64,
@@ -102,6 +103,7 @@ impl From<ente_assets::AssetDownloadProgress> for AssetDownloadProgress {
let display = ente_ensu::model::display_progress(&value);
Self {
label: value.asset_progress.label,
+ queued: value.queued,
downloaded_bytes: i64::try_from(value.batch_downloaded_bytes).unwrap_or(i64::MAX),
total_bytes: value
.batch_total_bytes
diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rs
index 04e1fb4421..41668f8093 100644
--- a/rust/bindings/uniffi/ensu/src/voice.rs
+++ b/rust/bindings/uniffi/ensu/src/voice.rs
@@ -229,7 +229,7 @@ impl DictationStream {
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
- self.session.lock().session.worker_exited(false)?;
+ self.session.lock().finish(false)?;
}
Ok(())
}
@@ -244,12 +244,21 @@ impl DictationStream {
let mut notified_stop = false;
let mut last_notification = Instant::now();
let mut on_event = |event: LiveEvent| {
+ if let LiveEvent::Deferred {
+ reason,
+ committed_through,
+ model_ready,
+ } = &event
+ {
+ let committed_ms = committed_through / 16;
+ let audio_ms = self.session.lock().session.accepted_samples() / 16;
+ ente_uniffi_log::info!(
+ "Dictation deferring remaining audio: reason={reason} committed_ms={committed_ms} audio_ms={audio_ms} model_ready={model_ready}"
+ );
+ }
let immediate = matches!(
&event,
- LiveEvent::Commit { .. }
- | LiveEvent::ReplaceTranscript(_)
- | LiveEvent::Deferred
- | LiveEvent::CaptureClosed
+ LiveEvent::Commit { .. } | LiveEvent::Deferred { .. } | LiveEvent::CaptureClosed
);
if matches!(&event, LiveEvent::Ready) {
let phase = self.session.lock().session.phase();
@@ -291,8 +300,14 @@ impl DictationStream {
state.session.observe_drained_audio(accepted, submitted)?;
}
}
- state.session.worker_exited(result.is_ok())?;
+ state.finish(result.is_ok())?;
+ let reason = state.session.stop_reason();
+ let audio_ms = state.session.accepted_samples() / 16;
+ let incomplete = state.session.incomplete();
drop(state);
+ ente_uniffi_log::info!(
+ "Dictation finished: reason={reason:?} audio_ms={audio_ms} incomplete={incomplete}"
+ );
self.stream.finish();
if !notified_stop {
callback.on_capture_closed();
@@ -386,7 +401,7 @@ mod tests {
stream.abandon(DictationStopReason::Interruption).unwrap();
assert!(owner.snapshot().worker_active);
assert!(DictationStream::new(owner.clone(), 16000, 1).is_err());
- owner.lock().session.worker_exited(false).unwrap();
+ owner.lock().finish(false).unwrap();
let value = owner.snapshot();
assert_eq!(
format!("{}{}", value.text, value.hypothesis),
@@ -436,7 +451,7 @@ mod tests {
}
}
#[test]
- fn replacement_matches_the_composer_preview_and_discard_rejects_late_text() {
+ fn completion_preserves_committed_text_and_discard_rejects_late_text() {
let owner = DictationSession::new(true);
let stream = DictationStream::new(owner.clone(), 16000, 1).unwrap();
event(
@@ -463,13 +478,24 @@ mod tests {
);
event(
&owner,
- LiveEvent::ReplaceTranscript("Complete recording".into()),
+ LiveEvent::Commit {
+ start: 16000,
+ end: 32000,
+ text: "corrected tail".into(),
+ },
);
- assert_eq!(owner.snapshot().text, "Complete recording");
+ assert_eq!(owner.snapshot().text, "partial corrected tail");
assert!(owner.snapshot().hypothesis.is_empty());
stream.abandon(DictationStopReason::Discard).unwrap();
- event(&owner, LiveEvent::ReplaceTranscript("late".into()));
- assert_eq!(owner.snapshot().text, "Complete recording");
+ event(
+ &owner,
+ LiveEvent::Commit {
+ start: 32000,
+ end: 48000,
+ text: "late".into(),
+ },
+ );
+ assert_eq!(owner.snapshot().text, "partial corrected tail");
assert!(matches!(owner.snapshot().phase, DictationPhase::Closed));
}
}
diff --git a/rust/bindings/uniffi/log/src/lib.rs b/rust/bindings/uniffi/log/src/lib.rs
index f7a521192c..a500dc6e5f 100644
--- a/rust/bindings/uniffi/log/src/lib.rs
+++ b/rust/bindings/uniffi/log/src/lib.rs
@@ -1,4 +1,4 @@
-pub use log::warn;
+pub use log::{info, warn};
#[macro_export]
macro_rules! setup {
diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.json
index 9280db1850..c9add6495a 100644
--- a/rust/checks/lint-exceptions/suppressions.json
+++ b/rust/checks/lint-exceptions/suppressions.json
@@ -27,6 +27,7 @@
"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/platform.rs": ["unsafe_code"],
+ "rust/crates/ensu/src/transcription/text.rs": ["clippy::expect_used"],
"rust/crates/exif/src/isobmff.rs": ["clippy::expect_used"],
"rust/crates/image/src/color_management.rs": ["clippy::expect_used"],
"rust/crates/location/src/binary.rs": ["clippy::expect_used"],
diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rs
index f7586beca6..9b571628da 100644
--- a/rust/crates/assets/src/download.rs
+++ b/rust/crates/assets/src/download.rs
@@ -86,7 +86,7 @@ pub struct DownloadTarget {
pub destination: PathBuf,
}
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
pub struct Progress {
pub label: String,
pub downloaded_bytes: u64,
diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rs
index bffde907e4..da39008c39 100644
--- a/rust/crates/assets/src/store.rs
+++ b/rust/crates/assets/src/store.rs
@@ -42,6 +42,7 @@ pub struct AssetFile {
pub struct AssetDownloadProgress {
pub asset_index: usize,
pub asset_progress: Progress,
+ pub queued: bool,
pub batch_downloaded_bytes: u64,
pub batch_total_bytes: Option<u64>,
pub batch_percentage: f64,
@@ -218,20 +219,32 @@ impl AssetStore {
.map(|asset| self.asset_lock(asset))
.collect::<Result<Vec<_>, _>>()?;
let progress = std::sync::Mutex::new(BatchProgress::new(assets.len(), on_progress));
- let jobs = assets
- .iter()
- .zip(locks)
- .enumerate()
- .map(|(asset_index, (asset, lock))| {
- self.download_asset(
- asset_index,
- asset,
- lock,
- &progress,
- cancellation.clone(),
- &import,
- )
- });
+ let progress = &progress;
+ let import = &import;
+ let cancellation = &cancellation;
+ let jobs =
+ assets
+ .iter()
+ .zip(locks)
+ .enumerate()
+ .map(|(asset_index, (asset, lock))| async move {
+ let result = self
+ .download_asset(
+ asset_index,
+ asset,
+ lock,
+ progress,
+ cancellation.clone(),
+ import,
+ )
+ .await;
+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]
+ progress
+ .lock()
+ .expect("progress lock")
+ .set_state(asset_index, AssetDownloadState::Complete);
+ result
+ });
let results = join_all(jobs).await;
if cancellation.is_cancelled() {
return Err(Error::Cancelled);
@@ -267,16 +280,16 @@ impl AssetStore {
F: FnMut(AssetDownloadProgress) + Send,
I: Fn(&Asset, &Path) -> Result<(), Error> + Sync,
{
+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]
+ progress
+ .lock()
+ .expect("progress lock")
+ .set_state(asset_index, AssetDownloadState::Queued);
let _guard = tokio::select! {
guard = lock.lock() => guard,
_ = cancellation.cancelled() => return Err(Error::Cancelled),
};
if self.is_downloaded(asset) {
- #[expect(
- clippy::expect_used,
- reason = "A panic unwinds the download; its local state is not reused"
- )]
- progress.lock().expect("progress lock").skip(asset_index);
return Ok(());
}
import(asset, &self.staging_dir(asset))?;
@@ -288,6 +301,11 @@ impl AssetStore {
slot = self.coordination.asset_slots.acquire() => slot.expect("asset semaphore closed"),
_ = cancellation.cancelled() => return Err(Error::Cancelled),
};
+ #[expect(clippy::expect_used, reason = "A panic unwinds the download")]
+ progress
+ .lock()
+ .expect("progress lock")
+ .set_state(asset_index, AssetDownloadState::Downloading);
self.downloader()?
.download(
@@ -481,9 +499,16 @@ fn normalized_root(root: PathBuf) -> PathBuf {
}
}
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum AssetDownloadState {
+ Queued,
+ Downloading,
+ Complete,
+}
+
struct BatchProgress<F> {
assets: Vec<Option<Progress>>,
- pending: Vec<bool>,
+ states: Vec<AssetDownloadState>,
callback: F,
}
@@ -494,37 +519,48 @@ where
fn new(asset_count: usize, callback: F) -> Self {
Self {
assets: vec![None; asset_count],
- pending: vec![true; asset_count],
+ states: vec![AssetDownloadState::Queued; asset_count],
callback,
}
}
- fn skip(&mut self, asset_index: usize) {
- self.pending[asset_index] = false;
+ fn set_state(&mut self, asset_index: usize, state: AssetDownloadState) {
+ self.states[asset_index] = state;
+ self.emit(
+ asset_index,
+ self.assets[asset_index].clone().unwrap_or_default(),
+ );
}
fn update(&mut self, asset_index: usize, asset_progress: Progress) {
- self.pending[asset_index] = false;
self.assets[asset_index] = Some(asset_progress.clone());
self.emit(asset_index, asset_progress);
}
- fn emit(&mut self, asset_index: usize, asset_progress: Progress) {
+ fn emit(&mut self, asset_index: usize, mut asset_progress: Progress) {
+ let queued = self.states.contains(&AssetDownloadState::Queued)
+ && !self.states.contains(&AssetDownloadState::Downloading);
+ if queued {
+ asset_progress.label = "Download queued".to_string();
+ } else if asset_progress.label.is_empty() {
+ asset_progress.label = "Preparing downloads".to_string();
+ }
let batch_downloaded_bytes = self
.assets
.iter()
.flatten()
.map(|progress| progress.downloaded_bytes)
.sum();
- let batch_total_bytes = (!self.pending.iter().any(|pending| *pending))
- .then(|| {
- self.assets
- .iter()
- .flatten()
- .map(|progress| progress.total_bytes)
- .try_fold(0u64, |total, value| total.checked_add(value?))
+ let batch_total_bytes = self
+ .assets
+ .iter()
+ .zip(&self.states)
+ .map(|(progress, state)| match progress {
+ Some(progress) => progress.total_bytes,
+ None if *state == AssetDownloadState::Complete => Some(0),
+ None => None,
})
- .flatten();
+ .try_fold(0u64, |total, value| total.checked_add(value?));
let batch_percentage = batch_total_bytes
.filter(|total| *total > 0)
.map(|total| ((batch_downloaded_bytes as f64 / total as f64) * 100.0).clamp(0.0, 100.0))
@@ -532,6 +568,7 @@ where
(self.callback)(AssetDownloadProgress {
asset_index,
asset_progress,
+ queued,
batch_downloaded_bytes,
batch_total_bytes,
batch_percentage,
@@ -890,24 +927,55 @@ mod tests {
Asset::files(key(&["models", "second"]), vec![file("model", b"second")]).unwrap();
stage_files(&store, &first, &[("model", b"first")]);
stage_files(&store, &second, &[("model", b"second")]);
- let mut latest_progress = None;
- let mut asset_progress = [0, 0];
-
- store
- .download(
- &[first.clone(), second.clone()],
- |progress| {
- asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;
- latest_progress = Some((
- progress.batch_downloaded_bytes,
- progress.batch_total_bytes,
- progress.batch_percentage,
- ));
- },
- CancellationToken::default(),
- )
+ let slots = store
+ .coordination
+ .asset_slots
+ .acquire_many(ASSET_CONCURRENCY as u32)
.await
.unwrap();
+ let second_lock = store.asset_lock(&second).unwrap();
+ let second_guard = second_lock.lock().await;
+ let (updates, mut receiver) = tokio::sync::mpsc::unbounded_channel();
+ let assets = [first.clone(), second.clone()];
+ let download = store.download(
+ &assets,
+ |progress| updates.send(progress).unwrap(),
+ CancellationToken::default(),
+ );
+ let mut asset_progress = [0, 0];
+ let observe_queue = async {
+ assert!(receiver.recv().await.unwrap().queued);
+ drop(slots);
+ let mut started = false;
+ loop {
+ let progress = receiver.recv().await.unwrap();
+ asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;
+ if !progress.queued {
+ started = true;
+ } else if progress.batch_downloaded_bytes == 5 {
+ assert!(started);
+ assert_eq!(progress.asset_progress.label, "Download queued");
+ break;
+ }
+ }
+ drop(second_guard);
+ };
+ let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
+ tokio::join!(download, observe_queue)
+ })
+ .await
+ .unwrap();
+ result.unwrap();
+ let mut latest_progress = None;
+ while let Ok(progress) = receiver.try_recv() {
+ assert!(!progress.queued);
+ asset_progress[progress.asset_index] = progress.asset_progress.downloaded_bytes;
+ latest_progress = Some((
+ progress.batch_downloaded_bytes,
+ progress.batch_total_bytes,
+ progress.batch_percentage,
+ ));
+ }
assert!(store.is_downloaded(&first));
assert!(store.is_downloaded(&second));
diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rs
index 34047a748e..0b523eb5ec 100644
--- a/rust/crates/ensu/src/config.rs
+++ b/rust/crates/ensu/src/config.rs
@@ -374,34 +374,21 @@ mod tests {
}
}
- #[test]
- fn live_voice_requires_more_than_four_gib() {
- let four_gib = 4 * 1024 * 1024 * 1024;
- for (memory, expected) in [
- (None, false),
- (Some(0), false),
- (Some(four_gib - 1), false),
- (Some(four_gib), false),
- (Some(four_gib + 1), true),
- (Some(8 * 1024 * 1024 * 1024), true),
- ] {
- assert_eq!(supports_live_voice(memory), expected, "RAM: {memory:?}");
- }
- }
-
#[test]
fn mobile_ram_ladder_enforces_inclusive_boundaries_and_saved_preferences() {
let gib = 1024 * 1024 * 1024;
for surface in [ModelRuntimeSurface::Android, ModelRuntimeSurface::Ios] {
- for (memory, expected) in [
- (None, "lfm-350m"),
- (Some(0), "lfm-350m"),
- (Some(4 * gib), "lfm-350m"),
- (Some(4 * gib + 1), "lfm-vl-1.6b"),
- (Some(8 * gib), "lfm-vl-1.6b"),
- (Some(8 * gib + 1), "gemma-4-e2b-q4km"),
+ for (memory, expected, live_voice) in [
+ (None, "lfm-350m", false),
+ (Some(0), "lfm-350m", false),
+ (Some(4 * gib - 1), "lfm-350m", false),
+ (Some(4 * gib), "lfm-350m", false),
+ (Some(4 * gib + 1), "lfm-vl-1.6b", true),
+ (Some(8 * gib), "lfm-vl-1.6b", true),
+ (Some(8 * gib + 1), "gemma-4-e2b-q4km", true),
] {
let policy = resolve_model_policy(surface, memory);
+ assert_eq!(supports_live_voice(memory), live_voice, "RAM: {memory:?}");
assert_eq!(policy.default_model.id, expected, "RAM: {memory:?}");
assert_eq!(policy.visible_models.len(), 1);
assert_eq!(policy.visible_models[0].id, expected);
diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rs
index 1bba6a7a0d..9eb6e536af 100644
--- a/rust/crates/ensu/src/model/mod.rs
+++ b/rust/crates/ensu/src/model/mod.rs
@@ -167,7 +167,9 @@ pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
pub fn display_progress(progress: &AssetDownloadProgress) -> ModelDownloadDisplay {
let total = progress.batch_total_bytes.filter(|total| *total > 0);
- let status = if let Some(total) = total {
+ let status = if progress.queued {
+ "Download queued".to_string()
+ } else if let Some(total) = total {
format!(
"Downloading... {} / {}",
format_bytes(progress.batch_downloaded_bytes),
diff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rs
index faa79cf67c..059da6020d 100644
--- a/rust/crates/ensu/src/transcription/audio.rs
+++ b/rust/crates/ensu/src/transcription/audio.rs
@@ -17,34 +17,35 @@ pub fn extract_speech_from_pcm16(
vad_model_path: impl AsRef<Path>,
input_sample_rate: u32,
pcm_le: &[u8],
- is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
if input_sample_rate == 0 {
return Err(TranscriptionError::InvalidAudio(
"input sample rate must be greater than zero".to_string(),
));
}
+ let samples = pcm16_samples(pcm_le)?;
+ extract_speech(vad_model_path, input_sample_rate, &samples)
+}
+
+pub(super) fn pcm16_samples(pcm_le: &[u8]) -> Result<Vec<f32>> {
if !pcm_le.len().is_multiple_of(2) {
return Err(TranscriptionError::InvalidAudio(
"PCM16 input must contain an even number of bytes".to_string(),
));
}
- let samples = pcm_le
+ Ok(pcm_le
.as_chunks::<2>()
.0
.iter()
.map(|chunk| i16::from_le_bytes(*chunk) as f32 / 32768.0)
- .collect::<Vec<_>>();
-
- extract_speech(vad_model_path, input_sample_rate, &samples, is_cancelled)
+ .collect())
}
fn extract_speech(
vad_model_path: impl AsRef<Path>,
input_sample_rate: u32,
samples: &[f32],
- is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
if samples.is_empty() {
return Ok(Vec::new());
@@ -54,7 +55,7 @@ fn extract_speech(
if !vad_model_path.is_file() {
return Err(TranscriptionError::VadNotDownloaded);
}
- let silero = SileroV4Vad::new(vad_model_path)?;
+ let silero = SileroV4Vad::new(vad_model_path, 0.5)?;
let mut vad = SmoothedVad::new(
Box::new(silero),
VAD_PREFILL_FRAMES,
@@ -62,14 +63,13 @@ fn extract_speech(
VAD_ONSET_FRAMES,
);
- extract_speech_with_vad(input_sample_rate, samples, &mut vad, is_cancelled)
+ extract_speech_with_vad(input_sample_rate, samples, &mut vad)
}
fn extract_speech_with_vad(
input_sample_rate: u32,
samples: &[f32],
vad: &mut dyn Vad,
- is_cancelled: &mut impl FnMut() -> bool,
) -> Result<Vec<f32>> {
let mut resampler = FrameResampler::new(
input_sample_rate as usize,
@@ -91,15 +91,7 @@ fn extract_speech_with_vad(
Err(error) => vad_error = Some(error),
}
};
- for chunk in samples.chunks(input_sample_rate as usize) {
- if is_cancelled() {
- return Ok(Vec::new());
- }
- resampler.push(chunk, &mut emit)?;
- }
- if is_cancelled() {
- return Ok(Vec::new());
- }
+ resampler.push(samples, &mut emit)?;
resampler.finish(&mut emit)?;
if let Some(error) = vad_error {
return Err(error.into());
@@ -231,10 +223,9 @@ mod tests {
use super::*;
#[test]
- fn vad_failure_and_cancellation_do_not_return_partial_recordings() {
+ fn vad_failure_does_not_return_partial_recording() {
struct FailsAfterSpeech {
frames: usize,
- fail: bool,
}
impl Vad for FailsAfterSpeech {
fn frame_size(&self) -> usize {
@@ -245,7 +236,7 @@ mod tests {
_: &[f32],
) -> std::result::Result<bool, transcribe_rs::TranscribeError> {
self.frames += 1;
- if self.frames == 1 || !self.fail {
+ if self.frames == 1 {
Ok(true)
} else {
Err(transcribe_rs::TranscribeError::Inference(
@@ -255,27 +246,11 @@ mod tests {
}
fn reset(&mut self) {}
}
- let mut vad = FailsAfterSpeech {
- frames: 0,
- fail: true,
- };
+ let mut vad = FailsAfterSpeech { frames: 0 };
assert!(
- matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad, &mut || false), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure"))
+ matches!(extract_speech_with_vad(16000, &vec![0.2; 1440], &mut vad), Err(TranscriptionError::Transcribe(detail)) if detail.contains("injected VAD failure"))
);
assert_eq!(vad.frames, 2);
- let mut vad = FailsAfterSpeech {
- frames: 0,
- fail: false,
- };
- let mut checks = 0;
- let speech = extract_speech_with_vad(16000, &vec![0.2; 32000], &mut vad, &mut || {
- checks += 1;
- checks > 1
- })
- .unwrap();
- assert!(speech.is_empty());
- assert_eq!(checks, 2);
- assert_eq!(vad.frames, 16000 / vad.frame_size());
}
fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {
@@ -295,6 +270,13 @@ mod tests {
#[test]
fn native_rate_preserves_samples_and_pads_the_last_frame() {
+ let pcm = [-32768_i16, -512, -1, 0, 1, 512, 32767];
+ let bytes: Vec<_> = pcm.iter().flat_map(|sample| sample.to_le_bytes()).collect();
+ assert_eq!(
+ pcm16_samples(&bytes).unwrap(),
+ pcm.map(|sample| f32::from(sample) / 32768.0)
+ );
+ assert!(pcm16_samples(&bytes[..bytes.len() - 1]).is_err());
let input: Vec<_> = (0..961).map(|i| i as f32 / 1000.0 - 0.5).collect();
let output = resample(&input, 16_000, 137);
diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rs
index 9d838e9933..d821a6cbab 100644
--- a/rust/crates/ensu/src/transcription/live.rs
+++ b/rust/crates/ensu/src/transcription/live.rs
@@ -7,16 +7,27 @@ use transcribe_rs::vad::Vad;
use super::{StreamNormalizer, Transcriber};
use crate::voice::{
AlignedWord, CaptureFeedResult, CaptureFormat, CaptureQueue, Result, RollingRecognizer,
- VoiceError,
+ TranscriptBoundary, VoiceError,
};
pub enum LiveEvent {
Ready,
- Audio { start: u64, end: u64, voiced: bool },
+ Audio {
+ start: u64,
+ end: u64,
+ voiced: bool,
+ },
Hypothesis(String),
- Commit { start: u64, end: u64, text: String },
- Deferred,
- ReplaceTranscript(String),
+ Commit {
+ start: u64,
+ end: u64,
+ text: String,
+ },
+ Deferred {
+ reason: VoiceError,
+ committed_through: u64,
+ model_ready: bool,
+ },
Coverage(u64),
Idle,
CaptureClosed,
@@ -121,8 +132,8 @@ impl LiveTranscription {
.load_model()
.map_err(|error| super::voice_error("Load transcription model", error))
},
- |request| transcriber.transcribe_window(&request.samples),
- |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),
+ |request| transcriber.transcribe_window(&request.samples, || self.queue.is_cancelled()),
+ |pcm| transcriber.transcribe_tail(pcm, || self.queue.is_cancelled()),
on_event,
)
}
@@ -132,7 +143,7 @@ impl LiveTranscription {
make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,
prepare: impl FnOnce() -> Result<()> + Send,
mut decode: impl FnMut(&crate::voice::DecodeWindow) -> Result<Vec<AlignedWord>> + Send,
- decode_recorded: impl FnOnce(&[u8]) -> Result<String>,
+ decode_recorded: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,
on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
) -> Result<()> {
let _capture = self.start()?;
@@ -144,6 +155,8 @@ impl LiveTranscription {
}
let mut normalizer = StreamNormalizer::new(self.queue.format())?;
let mut rolling = RollingRecognizer::default();
+ let mut boundary = TranscriptBoundary::default();
+ let mut provisional = Vec::new();
let mut pcm = super::recorded::RecordedPcm::default();
let mut accepted = 0_u64;
let mut deferred = false;
@@ -194,13 +207,20 @@ impl LiveTranscription {
},
on_event,
)?;
- pcm.push(&frame[..real])?;
+ pcm.push(&frame[..real], voiced)?;
if !deferred {
match rolling.feed(&frame[..real], voiced) {
Ok(()) => {}
Err(VoiceError::BacklogFull) => {
deferred = true;
- self.emit(LiveEvent::Deferred, on_event)?;
+ self.emit(
+ LiveEvent::Deferred {
+ reason: VoiceError::BacklogFull,
+ committed_through: boundary.end,
+ model_ready: prepared,
+ },
+ on_event,
+ )?;
}
Err(error) => return Err(error),
}
@@ -244,16 +264,25 @@ impl LiveTranscription {
on_event,
)?;
self.emit(
- LiveEvent::Coverage(update.covered_through),
+ LiveEvent::Coverage(update.boundary.end),
on_event,
)?;
+ boundary = update.boundary;
+ provisional = update.provisional;
}
Err(
- VoiceError::UnstableTranscript
- | VoiceError::IncompleteAudio,
+ reason @ (VoiceError::UnstableTranscript
+ | VoiceError::IncompleteAudio),
) => {
deferred = true;
- self.emit(LiveEvent::Deferred, on_event)?;
+ self.emit(
+ LiveEvent::Deferred {
+ reason,
+ committed_through: boundary.end,
+ model_ready: prepared,
+ },
+ on_event,
+ )?;
}
Err(error) => return Err(error),
}
@@ -304,13 +333,9 @@ impl LiveTranscription {
drop(vad);
drop(normalizer);
self.check()?;
- let text = decode_recorded(pcm.bytes())?;
+ self.finish_recording(&pcm, &boundary, &provisional, decode_recorded, on_event)?;
self.check()?;
- if text.trim().is_empty() {
- return Err(VoiceError::IncompleteAudio);
- }
- self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;
- self.emit(LiveEvent::Coverage(accepted), on_event)?;
+ self.emit(LiveEvent::Hypothesis(String::new()), on_event)?;
}
Ok(())
})
@@ -406,7 +431,8 @@ mod tests {
let (release, loading) = mpsc::sync_channel(1);
let mut accepted = 0;
let mut deferred = false;
- let mut replacement = None;
+ let mut replacement = String::new();
+ let mut decoded_samples = 0;
let mut coverage = 0;
stream
.run_with(
@@ -417,8 +443,14 @@ mod tests {
},
|_| panic!("Initialization exceeded the existing live backlog limit"),
|pcm| {
- assert_eq!(pcm, [0, 32].repeat(12 * 16000));
- Ok("All opening speech".into())
+ assert!(pcm.len() <= super::super::MAX_RECORDED_WINDOW_SAMPLES * 2);
+ assert!(pcm.as_chunks::<2>().0.iter().all(|frame| *frame == [0, 32]));
+ let first = decoded_samples == 0;
+ decoded_samples += pcm.len() / 2;
+ super::super::recorded::decoded_chunk(
+ pcm,
+ if first { "All opening" } else { " speech" },
+ )
},
&mut |event| {
match event {
@@ -434,8 +466,8 @@ mod tests {
release.send(()).unwrap();
}
}
- LiveEvent::Deferred => deferred = true,
- LiveEvent::ReplaceTranscript(text) => replacement = Some(text),
+ LiveEvent::Deferred { .. } => deferred = true,
+ LiveEvent::Commit { text, .. } => replacement.push_str(&text),
LiveEvent::Coverage(end) => coverage = end,
_ => {}
}
@@ -443,7 +475,8 @@ mod tests {
},
)
.unwrap();
- assert_eq!(replacement.as_deref(), Some("All opening speech"));
+ assert_eq!(replacement, "All opening speech");
+ assert!(decoded_samples >= 12 * 16000);
assert_eq!(coverage, 12 * 16000);
}
@@ -548,7 +581,7 @@ mod tests {
}
#[test]
- fn failed_live_decode_continues_capture_and_replays_every_sample_once() {
+ fn failed_initial_decode_recovers_all_audio_without_ending_capture() {
for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {
let stream = stream();
for _ in 0..3 {
@@ -565,11 +598,11 @@ mod tests {
|_| Err(failure.clone()),
|pcm| {
assert_eq!(pcm, [0, 32].repeat(9600 + 1001));
- Ok("Complete recording".into())
+ super::super::recorded::decoded_chunk(pcm, "Complete recording")
},
&mut |event| {
match event {
- LiveEvent::Deferred => {
+ LiveEvent::Deferred { .. } => {
assert!(!deferred);
assert!(!closed);
deferred = true;
@@ -580,7 +613,7 @@ mod tests {
assert!(deferred);
closed = true;
}
- LiveEvent::ReplaceTranscript(text) => {
+ LiveEvent::Commit { text, .. } => {
assert!(closed);
final_text.push(text);
}
@@ -596,29 +629,176 @@ mod tests {
}
}
+ #[test]
+ fn recovery_preserves_committed_prefix_and_cancel_keeps_the_visible_tail() {
+ use super::super::DictationState;
+
+ fn word(text: &str, start: u64, end: u64) -> AlignedWord {
+ AlignedWord {
+ text: text.into(),
+ start,
+ end,
+ }
+ }
+
+ for failure in [VoiceError::UnstableTranscript, VoiceError::IncompleteAudio] {
+ for outcome in 0..3 {
+ let stream = stream();
+ let (release, loading) = mpsc::sync_channel(1);
+ let mut state = DictationState::new(true);
+ state.session.begin_capture().unwrap();
+ let mut samples = [0, 32].repeat(192000);
+ samples[128000 * 2..128512 * 2].fill(0);
+ let mut fed = 0;
+ let mut calls = 0;
+ let mut recovered = 0;
+ let mut committed = false;
+ let mut deferred = false;
+ let expected_failure = failure.clone();
+ let recovery_failure =
+ VoiceError::ProcessingFailed("injected recovery failure".into());
+ let result = stream.run_with(
+ vad,
+ move || {
+ loading.recv_timeout(Duration::from_secs(3)).unwrap();
+ Ok(())
+ },
+ |_| {
+ calls += 1;
+ if calls > 3 {
+ return Err(failure.clone());
+ }
+ let mut words = vec![
+ word("Stable", 4000, 8000),
+ word("tentative", 64000, 68000),
+ word("tail", 72000, 76000),
+ ];
+ if calls == 3 {
+ words.push(word("later", 144000, 152000));
+ }
+ Ok(words)
+ },
+ |pcm| {
+ recovered += 1;
+ if recovered == 1 {
+ assert_eq!(pcm, &samples[..128512 * 2]);
+ if outcome == 1 {
+ stream.cancel();
+ }
+ Ok(vec![
+ word("Stable", 4000, 8000),
+ word("corrected", 64000, 68000),
+ word("tail", 72000, 76000),
+ ])
+ } else if outcome == 2 {
+ Err(recovery_failure.clone())
+ } else {
+ assert_eq!(recovered, 2);
+ assert_eq!(pcm, &samples[52000 * 2..]);
+ Ok(vec![
+ word("tail", 20000, 24000),
+ word("later", 92000, 100000),
+ word("ending", 138000, 140000),
+ ])
+ }
+ },
+ &mut |event| {
+ let feed_until = match &event {
+ LiveEvent::Idle if fed < 139264 => (fed + 3200).min(139264),
+ LiveEvent::Audio { end: 139264, .. } => {
+ release.send(()).unwrap();
+ fed
+ }
+ LiveEvent::Hypothesis(_) if !committed => fed + 8192,
+ LiveEvent::Commit { text, .. } if !committed => {
+ assert_eq!(text, "Stable");
+ committed = true;
+ fed + 8192
+ }
+ LiveEvent::Deferred {
+ reason,
+ committed_through,
+ model_ready,
+ } => {
+ assert_eq!(reason, &expected_failure);
+ assert_eq!(*committed_through, 8000);
+ assert!(*model_ready);
+ assert_eq!(state.text(), "Stable");
+ assert_eq!(state.hypothesis(), " tentative tail later");
+ deferred = true;
+ 192000
+ }
+ _ => fed,
+ };
+ while fed < feed_until {
+ let end = (fed + 3200).min(feed_until);
+ stream.feed(&samples[fed * 2..end * 2])?;
+ fed = end;
+ }
+ if matches!(event, LiveEvent::Deferred { .. }) {
+ stream.finish();
+ }
+ state.apply(event, || false)?;
+ Ok(LiveControl::Continue)
+ },
+ );
+ assert!(deferred && committed);
+ state.finish(result.is_ok()).unwrap();
+ match outcome {
+ 1 => {
+ assert_eq!(result, Err(VoiceError::Cancelled));
+ assert_eq!(state.text(), "Stable tentative tail later");
+ }
+ 2 => {
+ assert_eq!(result, Err(recovery_failure));
+ assert_eq!(state.text(), "Stable corrected tail later");
+ }
+ _ => {
+ result.unwrap();
+ assert_eq!(state.text(), "Stable corrected tail later ending");
+ }
+ }
+ assert!(state.hypothesis().is_empty());
+ assert_eq!(state.session.incomplete(), outcome != 0);
+ }
+ }
+ }
+
#[test]
fn incomplete_or_cancelled_recovery_cannot_publish_text_or_coverage() {
- for cancelled in [false, true] {
+ for (samples, cancelled) in [(1001, false), (1001, true), (32000, false), (32000, true)] {
let stream = stream();
- stream.feed(&[0, 32].repeat(1001)).unwrap();
+ for chunk in [0, 32].repeat(samples).chunks(3200 * 2) {
+ stream.feed(chunk).unwrap();
+ }
stream.finish();
let result = stream.run_with(
vad,
|| Ok(()),
- |_| Err(VoiceError::UnstableTranscript),
|_| {
+ if samples == 1001 {
+ Err(VoiceError::UnstableTranscript)
+ } else {
+ Ok(vec![AlignedWord {
+ text: "truncated prefix".into(),
+ start: 0,
+ end: 8000,
+ }])
+ }
+ },
+ |pcm| {
if cancelled {
stream.cancel();
- Ok("late result".into())
+ super::super::recorded::decoded_chunk(pcm, "late result")
} else {
- Ok(String::new())
+ super::super::recorded::decoded_chunk(pcm, "")
}
},
&mut |event| {
- assert!(!matches!(
- event,
- LiveEvent::ReplaceTranscript(_) | LiveEvent::Coverage(_)
- ));
+ assert!(!matches!(event, LiveEvent::Commit { .. }));
+ if let LiveEvent::Coverage(end) = event {
+ assert_eq!(end, 0);
+ }
Ok(LiveControl::Continue)
},
);
diff --git a/rust/crates/ensu/src/transcription/mod.rs b/rust/crates/ensu/src/transcription/mod.rs
index 37720cdb56..8be3b041e3 100644
--- a/rust/crates/ensu/src/transcription/mod.rs
+++ b/rust/crates/ensu/src/transcription/mod.rs
@@ -4,9 +4,14 @@ mod live;
mod normalize;
mod recorded;
mod session;
+mod text;
mod transcriber;
mod vad;
+pub(super) const ENDPOINT_PADDING_SAMPLES: usize = crate::voice::VOICE_SAMPLE_RATE as usize / 2;
+
+pub(super) const MAX_RECORDED_WINDOW_SAMPLES: usize = crate::voice::MAX_WINDOW_SAMPLES * 2;
+
pub use live::{LiveControl, LiveEvent, LiveTranscription};
pub(crate) use normalize::{StreamNormalizer, VAD_FRAME_SAMPLES};
pub use session::DictationState;
diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rs
index 133eeaf934..d076e2f51a 100644
--- a/rust/crates/ensu/src/transcription/recorded.rs
+++ b/rust/crates/ensu/src/transcription/recorded.rs
@@ -1,75 +1,192 @@
+use std::ops::Range;
use std::time::Duration;
use transcribe_rs::vad::Vad;
use super::{LiveControl, LiveEvent, LiveTranscription, StreamNormalizer, Transcriber};
-use crate::voice::{Result, VoiceError};
+use crate::voice::{AlignedWord, RecognitionUpdate, Result, TranscriptBoundary, VoiceError};
-const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 2) * 16_000;
+const MAX_RECORDED_SAMPLES: usize = (30 * 60 + 4) * 16_000;
-pub(super) fn transcribe_chunks(
- mut samples: &[f32],
- mut decode: impl FnMut(&[f32]) -> super::Result<Option<String>>,
-) -> super::Result<Option<String>> {
- use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE, transcript_separator};
+pub(super) fn chunk_end(samples: &[f32]) -> usize {
+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};
- let mut text = String::new();
- while !samples.is_empty() {
- let end = if samples.len() <= MAX_WINDOW_SAMPLES {
- samples.len()
- } else {
- let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);
- let start = limit - 2 * VOICE_SAMPLE_RATE as usize;
- let offset = samples[start..limit]
- .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()
- .0
- .iter()
- .enumerate()
- .map(|(index, frame)| {
- (
- index,
- frame.iter().map(|sample| sample * sample).sum::<f32>(),
- )
- })
- .min_by(|a, b| a.1.total_cmp(&b.1))
- .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);
- start + offset
- };
- let Some(chunk) = decode(&samples[..end])? else {
- return Ok(None);
- };
- let chunk = chunk.trim();
- text.push_str(transcript_separator(&text, chunk));
- text.push_str(chunk);
- samples = &samples[end..];
+ if samples.len() <= MAX_WINDOW_SAMPLES {
+ samples.len()
+ } else {
+ let limit = MAX_WINDOW_SAMPLES.min(samples.len() - VOICE_SAMPLE_RATE as usize);
+ let start = limit - 2 * VOICE_SAMPLE_RATE as usize;
+ let offset = samples[start..limit]
+ .as_chunks::<{ super::VAD_FRAME_SAMPLES }>()
+ .0
+ .iter()
+ .enumerate()
+ .map(|(index, frame)| {
+ (
+ index,
+ frame.iter().map(|sample| sample * sample).sum::<f32>(),
+ )
+ })
+ .min_by(|a, b| a.1.total_cmp(&b.1))
+ .map_or(0, |(index, _)| (index + 1) * super::VAD_FRAME_SAMPLES);
+ start + offset
}
- Ok(Some(text))
}
#[derive(Default)]
-pub(super) struct RecordedPcm(Vec<u8>);
+pub(super) struct RecordedPcm {
+ bytes: Vec<u8>,
+ speech: Vec<Range<usize>>,
+}
impl RecordedPcm {
- pub fn push(&mut self, samples: &[f32]) -> Result<()> {
- if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(self.0.len() / 2) {
+ pub fn push(&mut self, samples: &[f32], voiced: bool) -> Result<()> {
+ let start = self.bytes.len() / 2;
+ if samples.len() > MAX_RECORDED_SAMPLES.saturating_sub(start) {
return Err(VoiceError::BacklogFull);
}
- self.0.extend(samples.iter().flat_map(|sample| {
+ self.bytes.extend(samples.iter().flat_map(|sample| {
((sample * 32768.0).round().clamp(-32768.0, 32767.0) as i16).to_le_bytes()
}));
+ if voiced && !samples.is_empty() {
+ let end = start + samples.len();
+ if let Some(previous) = self.speech.last_mut().filter(|range| range.end == start) {
+ previous.end = end;
+ } else {
+ self.speech.push(start..end);
+ }
+ }
Ok(())
}
+ #[cfg(test)]
pub fn bytes(&self) -> &[u8] {
- &self.0
+ &self.bytes
+ }
+
+ pub fn transcribe(
+ &self,
+ boundary: &TranscriptBoundary,
+ provisional: &[AlignedWord],
+ mut decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,
+ mut on_update: impl FnMut(RecognitionUpdate) -> Result<()>,
+ ) -> Result<()> {
+ use crate::voice::{MAX_WINDOW_SAMPLES, VOICE_SAMPLE_RATE};
+
+ let total = self.bytes.len() / 2;
+ let mut boundary = boundary.clone();
+ let mut provisional = provisional.to_vec();
+ if self.speech.is_empty() && provisional.is_empty() {
+ boundary.end = total as u64;
+ return on_update(boundary.pending_update(provisional));
+ }
+ while boundary.end < total as u64 {
+ let start = boundary.start() as usize;
+ let available = (start + MAX_WINDOW_SAMPLES + VOICE_SAMPLE_RATE as usize).min(total);
+ let samples = super::audio::pcm16_samples(&self.bytes[start * 2..available * 2])
+ .map_err(|error| super::voice_error("Read recorded audio", error))?;
+ let normal_end = start + chunk_end(&samples);
+ let recovery_end = (start + super::MAX_RECORDED_WINDOW_SAMPLES).min(total);
+ let mut progressed = false;
+ let mut failure = VoiceError::IncompleteAudio;
+ for end in [normal_end, recovery_end] {
+ if end as u64 <= boundary.end {
+ continue;
+ }
+ let result = decode(&self.bytes[start * 2..end * 2]).and_then(|words| {
+ let update =
+ boundary.recorded_update(words, &provisional, end - start, end == total)?;
+ if update.commit.is_none()
+ && update.provisional.is_empty()
+ && !self.has_speech(boundary.end as usize..end)
+ {
+ let mut next = TranscriptBoundary::default();
+ next.end = if end == total {
+ end as u64
+ } else {
+ (end as u64 - VOICE_SAMPLE_RATE * 3 / 2).max(boundary.end)
+ };
+ return Ok(next.pending_update(Vec::new()));
+ }
+ Ok(update)
+ });
+ match result {
+ Ok(update) => {
+ provisional = update.provisional.clone();
+ if update.boundary.end <= boundary.end {
+ on_update(update)?;
+ failure = VoiceError::IncompleteAudio;
+ } else {
+ boundary = update.boundary.clone();
+ on_update(update)?;
+ if end == total {
+ if self.has_speech(
+ (boundary.end as usize + super::ENDPOINT_PADDING_SAMPLES)
+ .min(total)..total,
+ ) {
+ return Err(VoiceError::IncompleteAudio);
+ }
+ if boundary.end < total as u64 {
+ boundary.end = total as u64;
+ on_update(boundary.pending_update(provisional))?;
+ }
+ return Ok(());
+ }
+ progressed = true;
+ break;
+ }
+ }
+ Err(error @ VoiceError::IncompleteAudio) => {
+ failure = error;
+ }
+ Err(error) => {
+ if error != VoiceError::Cancelled {
+ on_update(boundary.pending_update(provisional))?;
+ }
+ return Err(error);
+ }
+ }
+ if normal_end == recovery_end {
+ break;
+ }
+ }
+ if !progressed {
+ on_update(boundary.pending_update(provisional))?;
+ return Err(failure);
+ }
+ }
+ Ok(())
+ }
+
+ fn has_speech(&self, range: Range<usize>) -> bool {
+ let index = self
+ .speech
+ .partition_point(|speech| speech.end <= range.start);
+ range.start < range.end
+ && self
+ .speech
+ .get(index)
+ .is_some_and(|speech| speech.start < range.end)
}
}
impl Drop for RecordedPcm {
fn drop(&mut self) {
use zeroize::Zeroize;
- self.0.zeroize();
+ self.bytes.zeroize();
+ }
+}
+
+#[cfg(test)]
+pub(super) fn decoded_chunk(pcm: &[u8], text: &str) -> Result<Vec<AlignedWord>> {
+ if text.is_empty() {
+ return Ok(Vec::new());
}
+ Ok(vec![AlignedWord {
+ text: text.into(),
+ start: (pcm.len() as u64 / 2).saturating_sub(1600),
+ end: pcm.len() as u64 / 2,
+ }])
}
impl LiveTranscription {
@@ -80,7 +197,7 @@ impl LiveTranscription {
) -> Result<()> {
self.run_recorded_with(
|| transcriber.live_vad(),
- |pcm| transcriber.transcribe_recording(pcm, || self.queue.is_cancelled()),
+ |pcm| transcriber.transcribe_tail(pcm, || self.queue.is_cancelled()),
on_event,
)
}
@@ -88,7 +205,7 @@ impl LiveTranscription {
fn run_recorded_with(
&self,
make_vad: impl FnOnce() -> Result<Box<dyn Vad>>,
- decode: impl FnOnce(&[u8]) -> Result<String>,
+ decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,
on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
) -> Result<()> {
let _capture = self.start()?;
@@ -97,7 +214,6 @@ impl LiveTranscription {
self.emit(LiveEvent::Ready, on_event)?;
let mut pcm = RecordedPcm::default();
let mut accepted = 0_u64;
- let mut heard_speech = false;
let mut notified_stop = false;
loop {
self.check()?;
@@ -107,7 +223,6 @@ impl LiveTranscription {
let voiced = vad.is_speech(frame).map_err(|error| {
super::voice_error("Detect speech in recorded audio", error)
})?;
- heard_speech |= voiced;
let start = accepted;
accepted += real as u64;
self.emit(
@@ -118,7 +233,7 @@ impl LiveTranscription {
},
on_event,
)?;
- pcm.push(&frame[..real])
+ pcm.push(&frame[..real], voiced)
};
if let Some(chunk) = chunk {
normalizer.push(&chunk, &mut emit)?;
@@ -142,15 +257,35 @@ impl LiveTranscription {
drop(vad);
drop(normalizer);
self.check()?;
- if heard_speech {
- let text = decode(pcm.bytes())?;
- self.check()?;
- if !text.trim().is_empty() {
- self.emit(LiveEvent::ReplaceTranscript(text), on_event)?;
- }
- }
- self.emit(LiveEvent::Coverage(accepted), on_event)?;
- Ok(())
+ self.finish_recording(&pcm, &TranscriptBoundary::default(), &[], decode, on_event)
+ }
+
+ pub(super) fn finish_recording(
+ &self,
+ pcm: &RecordedPcm,
+ boundary: &TranscriptBoundary,
+ provisional: &[AlignedWord],
+ mut decode: impl FnMut(&[u8]) -> Result<Vec<AlignedWord>>,
+ on_event: &mut impl FnMut(LiveEvent) -> Result<LiveControl>,
+ ) -> Result<()> {
+ pcm.transcribe(
+ boundary,
+ provisional,
+ |bytes| {
+ self.check()?;
+ let words = decode(bytes)?;
+ self.check()?;
+ Ok(words)
+ },
+ |update| {
+ self.check()?;
+ if let Some((start, end, text)) = update.commit {
+ self.emit(LiveEvent::Commit { start, end, text }, on_event)?;
+ }
+ self.emit(LiveEvent::Hypothesis(update.hypothesis), on_event)?;
+ self.emit(LiveEvent::Coverage(update.boundary.end), on_event)
+ },
+ )
}
}
@@ -167,41 +302,130 @@ mod tests {
let mut samples = vec![0.25; MAX_WINDOW_SAMPLES * 3 + 1];
let quiet = MAX_WINDOW_SAMPLES - 2 * 16000 + 10 * super::super::VAD_FRAME_SAMPLES;
samples[quiet..quiet + super::super::VAD_FRAME_SAMPLES].fill(0.0);
- let mut offset = 0;
- let mut decoded = 0;
- let text = transcribe_chunks(&samples, |chunk| {
- assert!(chunk.len() <= MAX_WINDOW_SAMPLES);
- assert!(chunk.len() >= 16000);
- assert_eq!(chunk, &samples[offset..offset + chunk.len()]);
- if offset == 0 {
- assert_eq!(chunk.len(), quiet + super::super::VAD_FRAME_SAMPLES);
- }
- offset += chunk.len();
- decoded += 1;
- Ok(Some(" again ".into()))
- })
- .unwrap()
- .unwrap();
- assert_eq!(offset, samples.len());
- assert_eq!(text.split_whitespace().count(), decoded);
+ let mut pcm = RecordedPcm::default();
+ pcm.push(&samples, true).unwrap();
let mut calls = 0;
- assert!(
- transcribe_chunks(&samples, |_| {
+ let mut previous_window_end = 0;
+ let mut covered = 0;
+ let boundary = std::cell::RefCell::new(TranscriptBoundary::default());
+ pcm.transcribe(
+ &TranscriptBoundary::default(),
+ &[],
+ |bytes| {
+ let boundary = boundary.borrow();
+ let start = boundary.start() as usize;
+ let end = start + bytes.len() / 2;
+ assert!(bytes.len() <= MAX_WINDOW_SAMPLES * 2);
+ if calls == 0 {
+ assert_eq!(end, quiet + super::super::VAD_FRAME_SAMPLES);
+ } else {
+ assert!(start < boundary.end as usize);
+ assert!(boundary.end < previous_window_end as u64);
+ }
+ previous_window_end = end;
calls += 1;
- Ok((calls == 1).then(|| "unfinished".into()))
- })
- .unwrap()
- .is_none()
+ let word_end = if end == samples.len() {
+ end
+ } else {
+ end - 32000
+ };
+ Ok(vec![AlignedWord {
+ text: "again".into(),
+ start: (word_end - start - 1600) as u64,
+ end: (word_end - start) as u64,
+ }])
+ },
+ |update| {
+ assert!(update.boundary.end > covered);
+ covered = update.boundary.end;
+ *boundary.borrow_mut() = update.boundary;
+ Ok(())
+ },
+ )
+ .unwrap();
+ assert_eq!(covered, samples.len() as u64);
+ assert!(calls > 3);
+ let mut calls = 0;
+ let mut pending = String::new();
+ assert_eq!(
+ pcm.transcribe(
+ &TranscriptBoundary::default(),
+ &[],
+ |bytes| {
+ calls += 1;
+ if calls == 1 {
+ decoded_chunk(bytes, "unfinished")
+ } else {
+ Err(VoiceError::Cancelled)
+ }
+ },
+ |update| {
+ assert!(update.commit.is_none());
+ pending = update.hypothesis;
+ Ok(())
+ },
+ ),
+ Err(VoiceError::Cancelled)
);
assert_eq!(calls, 2);
+ assert_eq!(pending, "unfinished");
+
+ let mut pcm = RecordedPcm::default();
+ pcm.push(&vec![0.0; MAX_WINDOW_SAMPLES], false).unwrap();
+ pcm.push(&vec![0.25; 16000], true).unwrap();
+ let mut decoded = 0;
+ let mut covered = 0;
+ assert_eq!(
+ pcm.transcribe(
+ &TranscriptBoundary::default(),
+ &[],
+ |pcm| {
+ decoded += 1;
+ decoded_chunk(pcm, "")
+ },
+ |update| {
+ assert!(update.commit.is_none() && update.hypothesis.is_empty());
+ covered = update.boundary.end;
+ Ok(())
+ },
+ ),
+ Err(VoiceError::IncompleteAudio)
+ );
+ assert_eq!(decoded, 2);
+ assert!(covered > 0 && covered <= MAX_WINDOW_SAMPLES as u64);
}
#[test]
- fn recorded_capacity_rejects_a_whole_frame_without_overwriting_audio() {
- let mut pcm = RecordedPcm(vec![0; (MAX_RECORDED_SAMPLES - 1) * 2]);
- pcm.push(&[0.25]).unwrap();
+ fn recorded_capacity_drains_resampler_and_rejects_overflow_atomically() {
+ let prefix = (30 * 60 - 2) * 16_000;
+ let mut pcm = RecordedPcm {
+ bytes: vec![0; prefix * 2],
+ speech: Vec::new(),
+ };
+ let mut normalizer = StreamNormalizer::new(CaptureFormat {
+ sample_rate: 47_999,
+ channels: 1,
+ })
+ .unwrap();
+ let input = vec![0; 4799 * 2];
+ let accepted = std::cell::Cell::new(0);
+ let mut emit = |frame: &[f32], real: usize| {
+ accepted.set(accepted.get() + real);
+ pcm.push(&frame[..real], true)
+ };
+ while accepted.get() < 2 * 16_000 {
+ normalizer.push(&input, &mut emit).unwrap();
+ }
+ for _ in 0..20 {
+ normalizer.push(&input, &mut emit).unwrap();
+ }
+ normalizer.finish(&mut emit).unwrap();
+ assert!(accepted.get() > 5 * 16_000);
+ assert_eq!(pcm.bytes().len(), (prefix + accepted.get()) * 2);
+ pcm.bytes.resize((MAX_RECORDED_SAMPLES - 1) * 2, 0);
+ pcm.push(&[0.25], true).unwrap();
assert_eq!(pcm.bytes().len(), MAX_RECORDED_SAMPLES * 2);
- assert_eq!(pcm.push(&[0.5]), Err(VoiceError::BacklogFull));
+ assert_eq!(pcm.push(&[0.5], true), Err(VoiceError::BacklogFull));
assert_eq!(
&pcm.bytes()[pcm.bytes().len() - 2..],
&8192_i16.to_le_bytes()
@@ -210,17 +434,21 @@ mod tests {
#[test]
fn recorded_completion_preserves_the_tail_and_cancellation_suppresses_results() {
- for cancelled in [false, true] {
+ use super::super::DictationState;
+
+ for (cancelled, incomplete) in [(false, false), (true, false), (false, true)] {
let stream = LiveTranscription::new(CaptureFormat {
sample_rate: 16000,
channels: 1,
})
.unwrap();
- stream.feed(&[0, 32].repeat(1001)).unwrap();
- stream.finish();
+ let audio = [0, 32].repeat(12 * 16000 + 1001);
+ let mut chunks = audio.chunks(6400);
+ let mut state = DictationState::new(false);
+ state.session.begin_capture().unwrap();
let mut closed = 0;
- let mut commits = Vec::new();
- let mut covered = None;
+ let mut decoded = 0;
+ let mut covered = 0;
let result = stream.run_recorded_with(
|| {
Ok(Box::new(EnergyVad::new(
@@ -229,35 +457,62 @@ mod tests {
)))
},
|pcm| {
- assert_eq!(pcm, [0, 32].repeat(1001));
+ assert!(
+ !pcm.is_empty()
+ && pcm.len() <= super::super::MAX_RECORDED_WINDOW_SAMPLES * 2
+ );
+ decoded += 1;
+ if decoded == 1 {
+ let end = pcm.len() as u64 / 2 - 32000;
+ return Ok(vec![AlignedWord {
+ text: "dictated words".into(),
+ start: end - 1600,
+ end,
+ }]);
+ }
if cancelled {
stream.cancel();
}
- Ok("dictated words".into())
+ decoded_chunk(pcm, if incomplete { "" } else { "continued" })
},
&mut |event| {
match event {
- LiveEvent::CaptureClosed => closed += 1,
- LiveEvent::ReplaceTranscript(text) => {
- assert_eq!(closed, 1);
- commits.push(text);
+ LiveEvent::Ready | LiveEvent::Idle => {
+ if let Some(chunk) = chunks.next() {
+ stream.feed(chunk)?;
+ } else {
+ stream.finish();
+ }
}
- LiveEvent::Hypothesis(_) => panic!("recorded mode must not emit previews"),
- LiveEvent::Coverage(end) => covered = Some(end),
+ LiveEvent::CaptureClosed => closed += 1,
+ LiveEvent::Commit { .. } => assert_eq!(closed, 1),
+ LiveEvent::Coverage(end) => covered = end,
_ => {}
}
+ state.apply(event, || false)?;
Ok(LiveControl::Continue)
},
);
assert_eq!(closed, 1);
- if cancelled {
- assert_eq!(result, Err(VoiceError::Cancelled));
- assert!(commits.is_empty());
- assert_eq!(covered, None);
+ assert_eq!(decoded, 2);
+ state.session.worker_exited(result.is_ok()).unwrap();
+ if cancelled || incomplete {
+ assert_eq!(
+ result,
+ Err(if cancelled {
+ VoiceError::Cancelled
+ } else {
+ VoiceError::IncompleteAudio
+ })
+ );
+ assert_eq!(state.text(), "dictated words");
+ assert!(covered > 0 && covered < audio.len() as u64 / 2);
+ assert!(state.session.incomplete());
} else {
result.unwrap();
- assert_eq!(commits, ["dictated words"]);
- assert_eq!(covered, Some(1001));
+ assert_eq!(state.text(), "dictated words continued");
+ assert_eq!(covered, audio.len() as u64 / 2);
+ assert!(!state.session.incomplete());
}
}
}
@@ -289,7 +544,7 @@ mod tests {
},
|pcm| {
assert_eq!(pcm, audio);
- Ok("Complete recorded transcript".into())
+ decoded_chunk(pcm, "Complete recorded transcript")
},
&mut |event| {
let idle = matches!(event, LiveEvent::Idle);
diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rs
index cee8f3ae32..c4755d36cd 100644
--- a/rust/crates/ensu/src/transcription/session.rs
+++ b/rust/crates/ensu/src/transcription/session.rs
@@ -32,6 +32,17 @@ impl DictationState {
)
}
+ pub fn finish(&mut self, successful: bool) -> voice::Result<()> {
+ self.session.worker_exited(successful)?;
+ self.text = super::text::filter_transcription_output(&format!(
+ "{}{}",
+ self.text,
+ self.hypothesis()
+ ));
+ self.hypothesis.clear();
+ Ok(())
+ }
+
pub fn apply(
&mut self,
event: LiveEvent,
@@ -70,16 +81,7 @@ impl DictationState {
self.text.push_str(&text);
self.hypothesis.clear();
}
- LiveEvent::ReplaceTranscript(text) => {
- if text.trim().is_empty() {
- return Err(voice::VoiceError::IncompleteAudio);
- }
- self.session.observe_recognition(true)?;
- self.text = text;
- self.hypothesis.clear();
- self.last_commit_end = u64::MAX;
- }
- LiveEvent::Deferred => self.session.defer_recognition(),
+ LiveEvent::Deferred { .. } => self.session.defer_recognition(),
LiveEvent::Coverage(end) => self.session.recognize_through(end)?,
LiveEvent::Idle => {
if self.session.silence_elapsed() && finish_if_empty() {
@@ -104,6 +106,56 @@ mod tests {
state.apply(event, || true).unwrap();
}
+ #[test]
+ fn cleanup_waits_for_completion_and_spans_commits_and_unfinished_hypotheses() {
+ for successful in [false, true] {
+ let mut state = DictationState::new(true);
+ state.session.begin_capture().unwrap();
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: 0,
+ end: 32000,
+ voiced: true,
+ },
+ );
+ event(
+ &mut state,
+ LiveEvent::Commit {
+ start: 0,
+ end: 16000,
+ text: "um I I".into(),
+ },
+ );
+ event(&mut state, LiveEvent::Hypothesis("I uh think".into()));
+ assert_eq!(state.text(), "um I I");
+ assert_eq!(state.hypothesis(), " I uh think");
+ if successful {
+ event(
+ &mut state,
+ LiveEvent::Commit {
+ start: 16000,
+ end: 32000,
+ text: "I uh think".into(),
+ },
+ );
+ event(&mut state, LiveEvent::Coverage(32000));
+ }
+ state
+ .session
+ .stop(if successful {
+ voice::StopReason::Done
+ } else {
+ voice::StopReason::Interruption
+ })
+ .unwrap();
+ state.finish(successful).unwrap();
+ assert_eq!(state.text(), "I think");
+ assert!(state.hypothesis().is_empty());
+ assert_eq!(state.session.incomplete(), !successful);
+ }
+ }
+
#[test]
fn queued_audio_is_processed_before_silence_can_stop_capture() {
let mut state = DictationState::new(false);
@@ -158,7 +210,7 @@ mod tests {
event(state, LiveEvent::Commit { start, end, text });
}
event(state, LiveEvent::Hypothesis(update.hypothesis));
- event(state, LiveEvent::Coverage(update.covered_through));
+ event(state, LiveEvent::Coverage(update.boundary.end));
}
for (words, expected) in [
@@ -199,13 +251,14 @@ mod tests {
.collect()
};
for i in 0..3 {
- rolling.feed(&vec![0.2; 16000], true).unwrap();
+ rolling.feed(&vec![0.2; 8000], i == 0).unwrap();
+ rolling.feed(&vec![0.2; 8000], false).unwrap();
event(
&mut state,
LiveEvent::Audio {
start: i * 16000,
end: (i + 1) * 16000,
- voiced: true,
+ voiced: i == 0,
},
);
let request = rolling.next_window().unwrap().unwrap();
diff --git a/rust/crates/ensu/src/transcription/text.rs b/rust/crates/ensu/src/transcription/text.rs
new file mode 100644
index 0000000000..781fbbd952
--- /dev/null
+++ b/rust/crates/ensu/src/transcription/text.rs
@@ -0,0 +1,81 @@
+use regex::{Regex, regex};
+use std::sync::LazyLock;
+
+const FILLER_WORDS: &[&str] = &[
+ "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha",
+ "ehh",
+];
+
+#[expect(
+ clippy::expect_used,
+ reason = "Filler words are escaped before insertion into the fixed regex"
+)]
+static FILLER_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
+ FILLER_WORDS
+ .iter()
+ .map(|word| {
+ Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word)))
+ .expect("escaped filler word forms a valid regex")
+ })
+ .collect()
+});
+
+pub fn filter_transcription_output(text: &str) -> String {
+ let mut filtered = text.to_string();
+
+ for pattern in FILLER_PATTERNS.iter() {
+ filtered = pattern.replace_all(&filtered, "").to_string();
+ }
+
+ filtered = collapse_stutters(&filtered);
+ filtered = regex!(r"\s{2,}").replace_all(&filtered, " ").to_string();
+ filtered.trim().to_string()
+}
+
+fn collapse_stutters(text: &str) -> String {
+ let words = text.split_whitespace().collect::<Vec<_>>();
+ if words.is_empty() {
+ return text.to_string();
+ }
+
+ let mut result = Vec::new();
+ let mut i = 0;
+
+ while i < words.len() {
+ let word = words[i];
+ let word_lower = word.to_lowercase();
+
+ if word_lower.len() <= 2 && word_lower.chars().all(char::is_alphabetic) {
+ let mut count = 1;
+ while i + count < words.len() && words[i + count].to_lowercase() == word_lower {
+ count += 1;
+ }
+
+ result.push(word);
+ i += if count >= 3 { count } else { 1 };
+ } else {
+ result.push(word);
+ i += 1;
+ }
+ }
+
+ result.join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::filter_transcription_output;
+
+ #[test]
+ fn removes_filler_words() {
+ assert_eq!(
+ filter_transcription_output("um hello uh world"),
+ "hello world"
+ );
+ }
+
+ #[test]
+ fn collapses_repeated_short_stutters() {
+ assert_eq!(filter_transcription_output("I I I think"), "I think");
+ }
+}
diff --git a/rust/crates/ensu/src/transcription/transcriber.rs b/rust/crates/ensu/src/transcription/transcriber.rs
index a23c23afed..392edb35c6 100644
--- a/rust/crates/ensu/src/transcription/transcriber.rs
+++ b/rust/crates/ensu/src/transcription/transcriber.rs
@@ -3,7 +3,7 @@ use std::sync::{Mutex, MutexGuard, PoisonError};
use transcribe_rs::onnx::Quantization;
use transcribe_rs::onnx::parakeet::{ParakeetModel, ParakeetParams, TimestampGranularity};
-use transcribe_rs::{OrtAccelerator, get_ort_accelerator};
+use transcribe_rs::{OrtAccelerator, TranscriptionResult, get_ort_accelerator};
use super::graph_cache::{GRAPHS, GraphCache, LoadPlan, SOURCES};
use crate::transcription::audio::extract_speech_from_pcm16;
@@ -28,7 +28,7 @@ impl Transcriber {
if !metadata.is_file() {
return Err(crate::voice::VoiceError::VadNotDownloaded);
}
- super::SileroV4Vad::new(&self.vad_model_path)
+ super::SileroV4Vad::new(&self.vad_model_path, 0.3)
.map(|vad| Box::new(vad) as Box<dyn transcribe_rs::vad::Vad>)
.map_err(|error| super::voice_error("Load voice activity model", error))
}
@@ -36,8 +36,12 @@ impl Transcriber {
pub(crate) fn transcribe_window(
&self,
samples: &[f32],
+ mut is_cancelled: impl FnMut() -> bool,
) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {
- use crate::voice::{AlignedWord, MAX_WINDOW_SAMPLES, VoiceError};
+ use crate::voice::{MAX_WINDOW_SAMPLES, VoiceError};
+ if is_cancelled() {
+ return Err(VoiceError::Cancelled);
+ }
if samples.is_empty()
|| samples.len() > MAX_WINDOW_SAMPLES
|| samples.iter().any(|s| !s.is_finite())
@@ -52,34 +56,10 @@ impl Transcriber {
true,
)
.map_err(|error| super::voice_error("Load transcription model", error))?;
- let result = model
- .transcribe_with(
- samples,
- &ParakeetParams {
- timestamp_granularity: Some(TimestampGranularity::Word),
- ..Default::default()
- },
- )
- .map_err(|error| super::voice_error("Transcribe live audio", error))?;
- result
- .segments
- .ok_or(VoiceError::UnstableTranscript)?
- .into_iter()
- .map(|word| {
- if !word.start.is_finite()
- || !word.end.is_finite()
- || word.end < word.start
- || word.start * 16000.0 > samples.len() as f32
- {
- return Err(VoiceError::UnstableTranscript);
- }
- Ok(AlignedWord {
- text: word.text,
- start: (word.start.max(0.0) * 16000.0).round() as u64,
- end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples.len() as u64),
- })
- })
- .collect()
+ let result = decode_window(model, samples, &mut is_cancelled)
+ .map_err(|error| super::voice_error("Transcribe live audio", error))?
+ .ok_or(VoiceError::Cancelled)?;
+ aligned_words(result, samples.len())
}
pub fn new(model_dir: impl Into<PathBuf>, vad_model_path: impl Into<PathBuf>) -> Self {
@@ -120,31 +100,8 @@ impl Transcriber {
}
pub fn transcribe(&self, input_sample_rate: u32, pcm_le: &[u8]) -> Result<String> {
- self.transcribe_with_cancel(input_sample_rate, pcm_le, || false)
- .map(Option::unwrap_or_default)
- }
-
- pub(crate) fn transcribe_recording(
- &self,
- pcm_le: &[u8],
- is_cancelled: impl FnMut() -> bool,
- ) -> crate::voice::Result<String> {
- self.transcribe_with_cancel(16_000, pcm_le, is_cancelled)
- .map_err(|error| super::voice_error("Transcribe recorded audio", error))?
- .ok_or(crate::voice::VoiceError::Cancelled)
- }
-
- fn transcribe_with_cancel(
- &self,
- input_sample_rate: u32,
- pcm_le: &[u8],
- mut is_cancelled: impl FnMut() -> bool,
- ) -> Result<Option<String>> {
- if is_cancelled() {
- return Ok(None);
- }
if pcm_le.is_empty() {
- return Ok(Some(String::new()));
+ return Ok(String::new());
}
if !self.model_dir.is_dir() {
return Err(TranscriptionError::NotDownloaded);
@@ -152,40 +109,56 @@ impl Transcriber {
if !self.vad_model_path.is_file() {
return Err(TranscriptionError::VadNotDownloaded);
}
-
- let speech = extract_speech_from_pcm16(
- &self.vad_model_path,
- input_sample_rate,
- pcm_le,
- &mut is_cancelled,
+ let speech = extract_speech_from_pcm16(&self.vad_model_path, input_sample_rate, pcm_le)?;
+ if speech.is_empty() {
+ return Ok(String::new());
+ }
+ let mut loaded = self.lock();
+ let model = ensure_loaded(
+ &mut loaded,
+ &self.model_dir,
+ self.graph_cache.as_ref(),
+ true,
+ )?;
+ let result = model.transcribe_with(
+ &speech,
+ &ParakeetParams {
+ timestamp_granularity: Some(TimestampGranularity::Segment),
+ ..Default::default()
+ },
)?;
+ Ok(super::text::filter_transcription_output(&result.text))
+ }
+
+ pub(crate) fn transcribe_tail(
+ &self,
+ pcm_le: &[u8],
+ mut is_cancelled: impl FnMut() -> bool,
+ ) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {
+ use crate::voice::VoiceError;
+
if is_cancelled() {
- return Ok(None);
+ return Err(VoiceError::Cancelled);
}
- if speech.is_empty() {
- return Ok(Some(String::new()));
+ let samples = super::audio::pcm16_samples(pcm_le)
+ .map_err(|error| super::voice_error("Read recorded audio", error))?;
+ if samples.is_empty() || samples.len() > super::MAX_RECORDED_WINDOW_SAMPLES {
+ return Err(VoiceError::InvalidAudio(
+ "Recorded audio exceeds the recognition window".into(),
+ ));
}
-
let mut loaded = self.lock();
let model = ensure_loaded(
&mut loaded,
&self.model_dir,
self.graph_cache.as_ref(),
false,
- )?;
- super::recorded::transcribe_chunks(&speech, |samples| {
- if is_cancelled() {
- return Ok(None);
- }
- let result = model.transcribe_with(
- samples,
- &ParakeetParams {
- timestamp_granularity: Some(TimestampGranularity::Segment),
- ..Default::default()
- },
- )?;
- Ok((!is_cancelled()).then_some(result.text))
- })
+ )
+ .map_err(|error| super::voice_error("Load transcription model", error))?;
+ let result = decode_window(model, &samples, &mut is_cancelled)
+ .map_err(|error| super::voice_error("Transcribe recorded audio", error))?
+ .ok_or(VoiceError::Cancelled)?;
+ aligned_words(result, samples.len())
}
fn lock(&self) -> MutexGuard<'_, Option<ParakeetModel>> {
@@ -193,6 +166,63 @@ impl Transcriber {
}
}
+fn decode_window(
+ model: &mut ParakeetModel,
+ samples: &[f32],
+ is_cancelled: &mut impl FnMut() -> bool,
+) -> Result<Option<TranscriptionResult>> {
+ if is_cancelled() {
+ return Ok(None);
+ }
+ let params = ParakeetParams {
+ timestamp_granularity: Some(TimestampGranularity::Word),
+ ..Default::default()
+ };
+ let mut result = model.transcribe_with(samples, &params)?;
+ if is_cancelled() {
+ return Ok(None);
+ }
+ let maximum = if samples.len() <= crate::voice::MAX_WINDOW_SAMPLES {
+ crate::voice::MAX_WINDOW_SAMPLES
+ } else {
+ super::MAX_RECORDED_WINDOW_SAMPLES
+ };
+ let padding = (maximum - samples.len()).min(super::ENDPOINT_PADDING_SAMPLES);
+ if result.text.trim().is_empty() && padding > 0 {
+ let mut padded = Vec::with_capacity(samples.len() + padding);
+ padded.extend_from_slice(samples);
+ padded.resize(samples.len() + padding, 0.0);
+ result = model.transcribe_with(&padded, &params)?;
+ }
+ Ok((!is_cancelled()).then_some(result))
+}
+
+fn aligned_words(
+ result: TranscriptionResult,
+ samples: usize,
+) -> crate::voice::Result<Vec<crate::voice::AlignedWord>> {
+ use crate::voice::{AlignedWord, VoiceError};
+ result
+ .segments
+ .ok_or(VoiceError::UnstableTranscript)?
+ .into_iter()
+ .map(|word| {
+ if !word.start.is_finite()
+ || !word.end.is_finite()
+ || word.end < word.start
+ || word.start * 16000.0 > samples as f32
+ {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ Ok(AlignedWord {
+ text: word.text,
+ start: (word.start.max(0.0) * 16000.0).round() as u64,
+ end: ((word.end.max(0.0) * 16000.0).round() as u64).min(samples as u64),
+ })
+ })
+ .collect()
+}
+
fn ensure_loaded<'a>(
loaded: &'a mut Option<ParakeetModel>,
model_dir: &Path,
diff --git a/rust/crates/ensu/src/transcription/vad.rs b/rust/crates/ensu/src/transcription/vad.rs
index 8869d0139c..b1beb87037 100644
--- a/rust/crates/ensu/src/transcription/vad.rs
+++ b/rust/crates/ensu/src/transcription/vad.rs
@@ -12,6 +12,7 @@ pub(crate) struct SileroV4Vad {
session: Session,
hidden: [f32; 128],
cell: [f32; 128],
+ threshold: f32,
}
fn inference(error: impl std::fmt::Display) -> TranscribeError {
@@ -19,7 +20,7 @@ fn inference(error: impl std::fmt::Display) -> TranscribeError {
}
impl SileroV4Vad {
- pub fn new(path: impl AsRef<Path>) -> Result<Self, TranscribeError> {
+ pub fn new(path: impl AsRef<Path>, threshold: f32) -> Result<Self, TranscribeError> {
let session = Session::builder()
.map_err(inference)?
.with_optimization_level(GraphOptimizationLevel::Level3)
@@ -34,6 +35,7 @@ impl SileroV4Vad {
session,
hidden: [0.0; 128],
cell: [0.0; 128],
+ threshold,
})
}
@@ -97,7 +99,7 @@ impl Vad for SileroV4Vad {
SILERO_V4_FRAME_SAMPLES
}
fn is_speech(&mut self, frame: &[f32]) -> Result<bool, TranscribeError> {
- Ok(self.speech_probability(frame)? > 0.5)
+ Ok(self.speech_probability(frame)? > self.threshold)
}
fn reset(&mut self) {
self.hidden.fill(0.0);
diff --git a/rust/crates/ensu/src/voice/mod.rs b/rust/crates/ensu/src/voice/mod.rs
index 2d466e5a66..3fa2d5a69b 100644
--- a/rust/crates/ensu/src/voice/mod.rs
+++ b/rust/crates/ensu/src/voice/mod.rs
@@ -9,9 +9,8 @@ pub(crate) use capture::CaptureQueue;
pub use capture::{CaptureFeedResult, CaptureFormat};
pub(crate) use composer::transcript_separator;
pub use composer::{ComposerAnchor, ComposerTransfer, VoiceInsertion, preview_insertion};
-#[cfg(test)]
pub(crate) use rolling::RecognitionUpdate;
-pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer};
+pub(crate) use rolling::{AlignedWord, DecodeWindow, RollingRecognizer, TranscriptBoundary};
pub use session::{StopReason, VoicePhase, VoiceSession};
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rs
index 25d06ae8bc..9c41692e15 100644
--- a/rust/crates/ensu/src/voice/rolling.rs
+++ b/rust/crates/ensu/src/voice/rolling.rs
@@ -31,7 +31,124 @@ pub(crate) struct DecodeWindow {
pub(crate) struct RecognitionUpdate {
pub commit: Option<(u64, u64, String)>,
pub hypothesis: String,
- pub covered_through: u64,
+ pub provisional: Vec<AlignedWord>,
+ pub boundary: TranscriptBoundary,
+}
+
+#[derive(Clone, Debug, Default)]
+pub(crate) struct TranscriptBoundary {
+ pub end: u64,
+ anchor: Option<AlignedWord>,
+}
+
+impl TranscriptBoundary {
+ pub fn start(&self) -> u64 {
+ self.anchor.as_ref().map_or(self.end, |anchor| {
+ self.end
+ .saturating_sub(LOOKBACK)
+ .min(anchor.start.saturating_sub(PRE_SPEECH))
+ })
+ }
+
+ pub fn recorded_update(
+ &self,
+ words: Vec<AlignedWord>,
+ provisional: &[AlignedWord],
+ samples: usize,
+ final_window: bool,
+ ) -> Result<RecognitionUpdate> {
+ let end = self.start() + samples as u64;
+ let words = absolute_words(words, self.start(), end)?;
+ let skip = self.overlap_end(&words)?;
+ if words[skip..].is_empty() {
+ return Ok(self.pending_update(provisional.to_vec()));
+ }
+ let words = merge_provisional(provisional, &words[skip..], self.start(), end)?;
+ let count = words
+ .iter()
+ .take_while(|word| {
+ word.end
+ <= if final_window {
+ end
+ } else {
+ end.saturating_sub(LOOKBACK)
+ }
+ })
+ .count();
+ if count > 0 && words[count - 1].end <= self.end {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ let boundary = if count == 0 {
+ self.clone()
+ } else {
+ let anchor = words[count - 1].clone();
+ Self {
+ end: anchor.end,
+ anchor: Some(anchor),
+ }
+ };
+ let commit = (boundary.end > self.end)
+ .then(|| (self.end, boundary.end, join_words(&words[..count])));
+ let provisional = words[count..].to_vec();
+ Ok(RecognitionUpdate {
+ commit,
+ hypothesis: join_words(&provisional),
+ provisional,
+ boundary,
+ })
+ }
+
+ pub fn pending_update(&self, provisional: Vec<AlignedWord>) -> RecognitionUpdate {
+ RecognitionUpdate {
+ commit: None,
+ hypothesis: join_words(&provisional),
+ provisional,
+ boundary: self.clone(),
+ }
+ }
+
+ fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {
+ let Some(anchor) = &self.anchor else {
+ return Ok(0);
+ };
+ if words.is_empty() {
+ return Ok(0);
+ }
+ let anchor_key = alignment_key(&anchor.text);
+ for same_text in [true, false] {
+ let mut matches = words.iter().enumerate().filter_map(|(i, word)| {
+ let duration = if same_text {
+ (word.end - word.start).min(anchor.end - anchor.start)
+ } else {
+ (word.end - word.start).max(anchor.end - anchor.start)
+ };
+ let overlap = word
+ .end
+ .min(anchor.end)
+ .saturating_sub(word.start.max(anchor.start));
+ (word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE
+ && word.end.abs_diff(anchor.end) <= ALIGNMENT_TOLERANCE
+ && (overlap * 2 > duration
+ || (same_text && duration == 0 && word.start == anchor.start))
+ && if same_text {
+ alignment_key(&word.text) == anchor_key
+ } else {
+ word.end <= self.end
+ })
+ .then_some(i + 1)
+ });
+ match (matches.next(), matches.next()) {
+ (Some(end), None) => return Ok(end),
+ (Some(_), Some(_)) => return Err(VoiceError::UnstableTranscript),
+ _ => {}
+ }
+ }
+ if words[0].start >= self.end {
+ Ok(0)
+ } else {
+ Err(VoiceError::UnstableTranscript)
+ }
+ }
}
struct Region {
@@ -145,29 +262,18 @@ impl RollingRecognizer {
if self.in_flight != Some((request.id, request.start, request.end, request.final_phrase)) {
return Err(VoiceError::StaleEvent);
}
- let mut current = words;
- let mut previous_start = 0;
- for word in &mut current {
- if word.text.trim().is_empty()
- || word.start < previous_start
- || word.end < word.start
- || word.end > request.end - request.start
- {
- return Err(VoiceError::UnstableTranscript);
- }
- previous_start = word.start;
- word.start += request.start;
- word.end += request.start;
- }
- let skip = self.overlap_end(&current)?;
+ let current = absolute_words(words, request.start, request.end)?;
+ let boundary = self.boundary();
+ let skip = boundary.overlap_end(&current)?;
let remaining = &current[skip..];
- let old_skip = self.overlap_end(&self.previous)?;
+ let old_skip = boundary.overlap_end(&self.previous)?;
let mut observations = vec![1; current.len()];
let agreed_prefix = remaining
.iter()
.zip(&self.previous[old_skip..])
.take_while(|(a, b)| {
- a.text == b.text && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE
+ alignment_key(&a.text) == alignment_key(&b.text)
+ && a.start.abs_diff(b.start) <= ALIGNMENT_TOLERANCE
})
.count();
for i in 0..agreed_prefix {
@@ -188,8 +294,13 @@ impl RollingRecognizer {
} else {
agreed
};
- if request.final_phrase && current.is_empty() {
- return Err(VoiceError::IncompleteAudio);
+ if request.final_phrase {
+ let last_word = remaining.last().ok_or(VoiceError::IncompleteAudio)?;
+ if self.regions.front().is_some_and(|region| {
+ last_word.end.saturating_add(VOICE_SAMPLE_RATE / 2) < region.last_voice
+ }) {
+ return Err(VoiceError::IncompleteAudio);
+ }
}
let commit = if commit_count > 0 {
let end = if request.final_phrase {
@@ -208,7 +319,8 @@ impl RollingRecognizer {
} else {
None
};
- let hypothesis = join_words(&remaining[commit_count..]);
+ let provisional = remaining[commit_count..].to_vec();
+ let hypothesis = join_words(&provisional);
self.previous = current;
self.observations = observations;
self.last_decoded = request.end;
@@ -227,11 +339,7 @@ impl RollingRecognizer {
if self.covered <= request.start + LOOKBACK {
return Err(VoiceError::UnstableTranscript);
}
- retain_from = self
- .covered
- .saturating_sub(LOOKBACK)
- .min(self.anchor.as_ref().map_or(self.covered, |w| w.start))
- .max(request.start);
+ retain_from = self.boundary().start().max(request.start);
if retain_from <= request.start {
return Err(VoiceError::UnstableTranscript);
}
@@ -247,27 +355,15 @@ impl RollingRecognizer {
Ok(RecognitionUpdate {
commit,
hypothesis,
- covered_through: self.covered,
+ provisional,
+ boundary: self.boundary(),
})
}
- fn overlap_end(&self, words: &[AlignedWord]) -> Result<usize> {
- let Some(anchor) = &self.anchor else {
- return Ok(0);
- };
- if words.is_empty() {
- return Ok(0);
- }
- let anchor_key = alignment_key(&anchor.text);
- let mut matches = words.iter().enumerate().filter_map(|(i, word)| {
- (alignment_key(&word.text) == anchor_key
- && word.start.abs_diff(anchor.start) <= ALIGNMENT_TOLERANCE)
- .then_some(i + 1)
- });
- match (matches.next(), matches.next()) {
- (Some(end), None) => Ok(end),
- (None, None) if words[0].start >= self.covered => Ok(0),
- _ => Err(VoiceError::UnstableTranscript),
+ fn boundary(&self) -> TranscriptBoundary {
+ TranscriptBoundary {
+ end: self.covered,
+ anchor: self.anchor.clone(),
}
}
@@ -284,6 +380,114 @@ impl RollingRecognizer {
}
}
+fn merge_provisional(
+ previous: &[AlignedWord],
+ current: &[AlignedWord],
+ start: u64,
+ end: u64,
+) -> Result<Vec<AlignedWord>> {
+ let mut paired = vec![None; previous.len()];
+ let mut matched = vec![false; current.len()];
+ for (old_index, old) in previous.iter().enumerate() {
+ if old.start < start || old.end > end {
+ continue;
+ }
+ let key = alignment_key(&old.text);
+ let mut candidates = current.iter().enumerate().filter_map(|(index, word)| {
+ let same = alignment_key(&word.text) == key;
+ let duration = if same {
+ (word.end - word.start).min(old.end - old.start)
+ } else {
+ (word.end - word.start).max(old.end - old.start)
+ };
+ let overlap = word
+ .end
+ .min(old.end)
+ .saturating_sub(word.start.max(old.start));
+ (word.start.abs_diff(old.start) <= ALIGNMENT_TOLERANCE
+ && (same || word.end.abs_diff(old.end) <= ALIGNMENT_TOLERANCE)
+ && (overlap * 2 > duration || (same && duration == 0 && word.start == old.start)))
+ .then_some(index)
+ });
+ match (candidates.next(), candidates.next()) {
+ (Some(index), None) if !matched[index] => {
+ matched[index] = true;
+ paired[old_index] = Some(index);
+ }
+ (None, None) => {}
+ _ => return Err(VoiceError::UnstableTranscript),
+ }
+ }
+ let anchors = paired.clone();
+ for index in 1..previous.len() {
+ let (Some(left), Some(right)) = (anchors[index - 1], anchors[index]) else {
+ continue;
+ };
+ if right != left + 1
+ || alignment_key(&previous[index - 1].text) != alignment_key(&current[left].text)
+ || alignment_key(&previous[index].text) != alignment_key(&current[right].text)
+ {
+ continue;
+ }
+ let mut old_index = index + 1;
+ let mut new_index = right + 1;
+ while let (Some(old), Some(word)) = (previous.get(old_index), current.get(new_index)) {
+ if old.start < start
+ || old.end > end
+ || alignment_key(&old.text) != alignment_key(&word.text)
+ {
+ break;
+ }
+ if paired[old_index].is_some_and(|paired| paired != new_index)
+ || (paired[old_index].is_none() && matched[new_index])
+ {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ paired[old_index] = Some(new_index);
+ matched[new_index] = true;
+ old_index += 1;
+ new_index += 1;
+ }
+ }
+ let mut retained = Vec::new();
+ for (old, paired) in previous.iter().zip(paired) {
+ if paired.is_some() {
+ continue;
+ }
+ if current.iter().any(|word| {
+ (word.start < old.end && old.start < word.end)
+ || (word.start == old.start && (word.end == word.start || old.end == old.start))
+ }) {
+ return Err(if old.start < start || old.end > end {
+ VoiceError::IncompleteAudio
+ } else {
+ VoiceError::UnstableTranscript
+ });
+ }
+ retained.push(old.clone());
+ }
+ retained.extend_from_slice(current);
+ retained.sort_by_key(|word| word.start);
+ Ok(retained)
+}
+
+fn absolute_words(mut words: Vec<AlignedWord>, start: u64, end: u64) -> Result<Vec<AlignedWord>> {
+ let mut previous_start = 0;
+ for word in &mut words {
+ if word.text.trim().is_empty()
+ || word.start < previous_start
+ || word.end < word.start
+ || word.end > end - start
+ {
+ return Err(VoiceError::UnstableTranscript);
+ }
+ previous_start = word.start;
+ word.start += start;
+ word.end += start;
+ }
+ Ok(words)
+}
+
fn alignment_key(text: &str) -> String {
let lexical = text.trim().trim_matches([
'.', ',', '!', '?', ':', ';', '…', '。', ',', '!', '?', '؛', '،', '।', '॥', '¿', '¡',
@@ -321,9 +525,13 @@ mod tests {
fn provisional_revisions_need_repeated_right_context_before_commit() {
let mut rolling = RollingRecognizer::default();
for index in 0..5 {
- rolling.feed(&vec![0.1; 8000], true).unwrap();
+ rolling.feed(&vec![0.1; 8000], index == 0).unwrap();
let request = rolling.next_window().unwrap().unwrap();
- let first = if index < 2 { "Friday" } else { "Saturday" };
+ let first = match index {
+ 0 | 1 => "Friday",
+ 3 => "Saturday,",
+ _ => "Saturday",
+ };
let update = rolling
.complete(
&request,
@@ -342,13 +550,36 @@ mod tests {
assert_eq!(update.hypothesis, "at noon");
}
}
+ rolling.finish();
+ let request = rolling.next_window().unwrap().unwrap();
+ for words in [Vec::new(), vec![aligned("Saturday", 0, 1000)]] {
+ assert!(matches!(
+ rolling.complete(&request, words),
+ Err(VoiceError::IncompleteAudio)
+ ));
+ assert_eq!(rolling.covered(), 1000);
+ assert!(!rolling.is_settled());
+ }
+ let update = rolling
+ .complete(
+ &request,
+ vec![
+ aligned("Saturday", 0, 1000),
+ aligned("at", 1000, 2000),
+ aligned("noon", 2000, 3000),
+ ],
+ )
+ .unwrap();
+ assert_eq!(update.commit.unwrap(), (1000, 40000, "at noon".into()));
+ assert!(update.hypothesis.is_empty());
+ assert!(rolling.is_settled());
}
#[test]
fn repeated_unfinished_tail_stays_revisable_until_endpoint() {
let mut rolling = RollingRecognizer::default();
- for _ in 0..6 {
- rolling.feed(&vec![0.1; 8000], true).unwrap();
+ for index in 0..6 {
+ rolling.feed(&vec![0.1; 8000], index == 0).unwrap();
let request = rolling.next_window().unwrap().unwrap();
let update = rolling
.complete(
@@ -386,39 +617,252 @@ mod tests {
#[test]
fn overlap_ignores_boundary_punctuation_but_preserves_acoustic_identity() {
- let rolling = RollingRecognizer {
+ let boundary = TranscriptBoundary {
anchor: Some(aligned("You.", 16000, 20000)),
- covered: 20000,
- ..RollingRecognizer::default()
+ end: 20000,
};
assert_eq!(
- rolling.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),
+ boundary.overlap_end(&[aligned("you,", 16000, 20000), aligned("you", 22000, 26000)]),
Ok(1)
);
assert_eq!(
- rolling.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),
+ boundary.overlap_end(&[aligned("to", 14000, 18000), aligned("you", 16000, 20000)]),
+ Ok(2)
+ );
+ assert_eq!(
+ boundary.overlap_end(&[aligned("you", 16000, 18000), aligned("you", 18000, 20000)]),
Err(VoiceError::UnstableTranscript)
);
assert_eq!(
- rolling.overlap_end(&[aligned("you", 12000, 20000)]),
+ boundary.overlap_end(&[aligned("you", 12000, 20000)]),
Err(VoiceError::UnstableTranscript)
);
+ assert_eq!(boundary.overlap_end(&[aligned("not", 16000, 20000)]), Ok(1));
assert_eq!(
- rolling.overlap_end(&[aligned("not", 16000, 20000)]),
+ boundary.overlap_end(&[aligned("to", 14000, 18000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ let repeated = [aligned("no", 0, 1600), aligned("no", 1600, 3200)];
+ assert!(matches!(
+ merge_provisional(&repeated, &[aligned("no", 0, 3200)], 0, 40000),
+ Err(VoiceError::UnstableTranscript)
+ ));
+ assert_eq!(
+ join_words(
+ &merge_provisional(
+ &[aligned("no", 0, 1600)],
+ &[aligned("no", 1600, 3200)],
+ 0,
+ 40000
+ )
+ .unwrap()
+ ),
+ "no no"
+ );
+ assert!(matches!(
+ merge_provisional(
+ &[aligned("a", 16000, 16000)],
+ &[aligned("the", 16000, 16000)],
+ 0,
+ 40000
+ ),
Err(VoiceError::UnstableTranscript)
+ ));
+ let retained = merge_provisional(
+ &[
+ aligned("earlier", 16000, 20000),
+ aligned("later", 100000, 104000),
+ ],
+ &[aligned("corrected", 40000, 44000)],
+ 0,
+ 80000,
+ )
+ .unwrap();
+ assert_eq!(join_words(&retained), "earlier corrected later");
+ assert!(matches!(
+ merge_provisional(
+ &[aligned("hello", 100, 300)],
+ &[aligned("hello", 100, 200)],
+ 0,
+ 200,
+ ),
+ Err(VoiceError::IncompleteAudio)
+ ));
+ let compressed = [
+ aligned("campuses", 464320, 473280),
+ aligned("easier", 473280, 478400),
+ aligned("to", 478400, 479680),
+ aligned("reach", 479680, 482240),
+ aligned("than", 482240, 483040),
+ ];
+ let revised = [
+ aligned("campuses,", 464480, 474720),
+ aligned("easier", 474720, 481120),
+ aligned("to", 481120, 483680),
+ aligned("reach", 483680, 487520),
+ aligned("than", 487520, 490080),
+ ];
+ assert_eq!(
+ join_words(&merge_provisional(&compressed, &revised, 440320, 581120).unwrap()),
+ "campuses, easier to reach than"
+ );
+ let omitted = [&revised[..3], &revised[4..]].concat();
+ assert!(
+ merge_provisional(&compressed, &omitted, 440320, 581120)
+ .map_or(true, |words| join_words(&words).contains("reach"))
);
assert_ne!(alignment_key("don't"), alignment_key("do"));
assert_ne!(alignment_key("कि"), alignment_key("क"));
assert_ne!(alignment_key("-5"), alignment_key("5"));
- let rolling = RollingRecognizer {
+ let boundary = TranscriptBoundary {
+ anchor: Some(aligned("two", 16000, 18560)),
+ end: 18560,
+ };
+ for revised in ["two", "to"] {
+ assert_eq!(
+ boundary
+ .recorded_update(
+ vec![
+ aligned(revised, 16000, 18560),
+ aligned("two", 18560, 21120),
+ aligned("people", 21120, 25000),
+ ],
+ &[],
+ 32000,
+ true,
+ )
+ .unwrap()
+ .commit
+ .unwrap()
+ .2,
+ "two people"
+ );
+ }
+ let boundary = TranscriptBoundary {
+ anchor: Some(aligned("a", 16000, 16000)),
+ end: 16000,
+ };
+ assert_eq!(boundary.overlap_end(&[aligned("a", 16000, 16000)]), Ok(1));
+ assert_eq!(boundary.overlap_end(&[aligned("the", 16000, 16000)]), Ok(0));
+ assert_eq!(
+ boundary.overlap_end(&[aligned("a", 16000, 16000), aligned("a", 16000, 16000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ let boundary = TranscriptBoundary {
+ anchor: Some(aligned("going", 16000, 21000)),
+ end: 21000,
+ };
+ assert_eq!(
+ boundary.overlap_end(&[aligned("gonna", 16000, 23000)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ assert_eq!(
+ boundary.overlap_end(&[aligned("going", 16000, 23000)]),
+ Ok(1)
+ );
+ let boundary = TranscriptBoundary {
+ anchor: Some(aligned("are", 338400, 339200)),
+ end: 339200,
+ };
+ assert_eq!(
+ boundary.overlap_end(&[aligned("are", 338080, 340640)]),
+ Ok(1)
+ );
+ assert_eq!(
+ boundary.overlap_end(&[aligned("were", 338080, 340640)]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ assert_eq!(
+ boundary.overlap_end(&[
+ aligned("are", 338080, 340640),
+ aligned("are", 338400, 339200),
+ ]),
+ Err(VoiceError::UnstableTranscript)
+ );
+ let boundary = TranscriptBoundary {
anchor: Some(aligned("Café!", 16000, 20000)),
- covered: 20000,
- ..RollingRecognizer::default()
+ end: 20000,
+ };
+ assert_eq!(
+ boundary.overlap_end(&[aligned("café", 16000, 20000)]),
+ Ok(1)
+ );
+ assert_eq!(
+ boundary.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),
+ Ok(1)
+ );
+ let boundary = TranscriptBoundary {
+ end: 56000,
+ anchor: Some(aligned("You.", 50000, 56000)),
};
- assert_eq!(rolling.overlap_end(&[aligned("café", 16000, 20000)]), Ok(1));
+ assert_eq!(boundary.start(), 32000);
assert_eq!(
- rolling.overlap_end(&[aligned("cafe\u{301}", 16000, 20000)]),
+ boundary
+ .recorded_update(
+ vec![
+ aligned("before", 0, 8000),
+ aligned("you,", 18000, 24000),
+ aligned("you", 26000, 30000),
+ aligned("again", 30000, 34000),
+ ],
+ &[],
+ 40000,
+ true,
+ )
+ .unwrap()
+ .commit
+ .unwrap()
+ .2,
+ "you again"
+ );
+ assert!(matches!(
+ boundary.recorded_update(vec![aligned("different", 12000, 24000)], &[], 40000, true),
+ Err(VoiceError::UnstableTranscript)
+ ));
+ assert!(matches!(
+ boundary.recorded_update(vec![aligned("you", 18000, 50000)], &[], 40000, true),
Err(VoiceError::UnstableTranscript)
+ ));
+ let provisional = [aligned("you", 58000, 62000), aligned("again", 62000, 66000)];
+ for words in [Vec::new(), vec![aligned("you", 18000, 24000)]] {
+ let update = boundary
+ .recorded_update(words, &provisional, 40000, true)
+ .unwrap();
+ assert!(update.commit.is_none());
+ assert_eq!(update.boundary.end, boundary.end);
+ assert_eq!(update.hypothesis, "you again");
+ assert_eq!(join_words(&update.provisional), "you again");
+ }
+ assert_eq!(
+ TranscriptBoundary {
+ end: 56000,
+ anchor: None
+ }
+ .start(),
+ 56000
+ );
+ let boundary = TranscriptBoundary {
+ end: 403520,
+ anchor: Some(aligned("Americans", 379200, 403520)),
+ };
+ assert_eq!(boundary.start(), 363200);
+ assert_eq!(
+ boundary
+ .recorded_update(
+ vec![
+ aligned("America,", 18400, 40160),
+ aligned("ask", 40160, 51680),
+ aligned("Americans", 70000, 94000),
+ ],
+ &[],
+ 128000,
+ true,
+ )
+ .unwrap()
+ .commit
+ .unwrap()
+ .2,
+ "ask Americans"
);
}
@@ -490,22 +934,32 @@ mod tests {
#[test]
fn failed_alignment_keeps_audio_and_does_not_claim_completion() {
- let mut rolling = RollingRecognizer::default();
- rolling.feed(&vec![0.1; 8000], true).unwrap();
- let request = rolling.next_window().unwrap().unwrap();
- assert!(
- rolling
- .complete(
- &request,
- vec![AlignedWord {
- text: "bad".into(),
- start: 0,
- end: 9000
- }]
- )
- .is_err()
- );
- assert_eq!(rolling.retained_samples(), 8000);
- assert_eq!(rolling.covered(), 0);
+ for (samples, finished, word_end, error) in [
+ (8000, false, 9000, VoiceError::UnstableTranscript),
+ (32000, true, 8000, VoiceError::IncompleteAudio),
+ ] {
+ let mut rolling = RollingRecognizer::default();
+ rolling.feed(&vec![0.1; samples], true).unwrap();
+ if finished {
+ rolling.finish();
+ }
+ let request = rolling.next_window().unwrap().unwrap();
+ assert_eq!(
+ rolling
+ .complete(
+ &request,
+ vec![AlignedWord {
+ text: "prefix".into(),
+ start: 0,
+ end: word_end,
+ }]
+ )
+ .unwrap_err(),
+ error
+ );
+ assert_eq!(rolling.retained_samples(), samples);
+ assert_eq!(rolling.covered(), 0);
+ assert!(!rolling.is_settled());
+ }
}
}