💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Preserve dictation audio tails and transcript spacing
setalp1aa356015dd4
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.ktnew file mode 100644index 0000000000..19b5e4ebce--- /dev/null+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt@@ -0,0 +1,21 @@+package io.ente.ensu.chat++import java.io.IOException++internal fun captureMicrophoneAudio(+ isStopped: () -> Boolean,+ read: () -> Int,+ isInterrupted: () -> Boolean,+ deliver: (Int) -> Boolean,+) {+ while (!isStopped()) {+ val count = read()+ if (count <= 0) {+ if (isStopped()) return+ throw IOException("Microphone read failed")+ }+ // Stop changes the route too. Preserve a successful final read while draining.+ if (isInterrupted() && !isStopped()) throw IOException("Microphone interrupted")+ if (!deliver(count)) return+ }+}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex 52b58d1407..a0fcd2ee21 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -12,6 +12,7 @@ import androidx.compose.runtime.mutableStateOfimport androidx.compose.runtime.setValueimport io.ente.ensu.assets.AssetStoreimport io.ente.ensu.bindings.DictationException+import io.ente.ensu.bindings.DictationFeedResultimport io.ente.ensu.bindings.DictationInsertionimport io.ente.ensu.bindings.DictationPhaseimport io.ente.ensu.bindings.DictationSession@@ -94,7 +95,7 @@ internal class VoiceTranscriptionController(private var stream: DictationStream? = nullprivate var capture: LiveMicrophone? = nullprivate var work: Job? = null- private var finishingWatchdog: Job? = null+ private var finishingNoticeJob: Job? = nullprivate var attemptFinished = falseprivate var insertion: DictationInsertion? = nullprivate var foreground = true@@ -112,6 +113,9 @@ internal class VoiceTranscriptionController(var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)private set+ var isFinishingSlow by mutableStateOf(false)+ private set+var snapshot: DictationSnapshot? by mutableStateOf(null)private set@@ -133,6 +137,7 @@ internal class VoiceTranscriptionController(generation++initialRecordPending = trueattemptFinished = false+ isFinishingSlow = falsethis.insertion = insertionsession = DictationSession(live = usesLiveDictation)snapshot = session?.snapshot()@@ -289,8 +294,8 @@ internal class VoiceTranscriptionController(withContext(NonCancellable) { ownedCapture?.stopAndDrain() }hasActiveWork = falseif (token == generation) {- finishingWatchdog?.cancel()- finishingWatchdog = null+ finishingNoticeJob?.cancel()+ finishingNoticeJob = nullcapture = nullstream = nullattemptFinished = true@@ -319,6 +324,7 @@ internal class VoiceTranscriptionController(else -> VoiceInputState.Error("Voice input stopped.")}active = false+ isFinishingSlow = falsesnapshot = nullsession = nullinsertion = null@@ -336,10 +342,10 @@ internal class VoiceTranscriptionController(!active ||stream !== finishingStream ||!owner.snapshot().workerActive ||- finishingWatchdog != null+ finishingNoticeJob != null)return- finishingWatchdog = scope.launch {+ finishingNoticeJob = scope.launch {delay(10_000)if (token != generation ||@@ -348,12 +354,7 @@ internal class VoiceTranscriptionController(!owner.snapshot().workerActive)return@launch- capture?.stop()- runCatching { finishingStream.abandon(DictationStopReason.FAILURE) }- work?.cancel()- snapshot = owner.snapshot()- notice =- "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."+ isFinishingSlow = true}}@@ -382,6 +383,8 @@ internal class VoiceTranscriptionController(capture?.stop()runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }work?.cancel()+ finishingNoticeJob?.cancel()+ isFinishingSlow = falsenotice = messagestate = VoiceInputState.Transcribing}@@ -390,12 +393,13 @@ internal class VoiceTranscriptionController(awaitingPermission = falsestartOnForeground = falsegeneration++- finishingWatchdog?.cancel()- finishingWatchdog = null+ finishingNoticeJob?.cancel()+ finishingNoticeJob = nullcapture?.stop()runCatching { stream?.abandon(DictationStopReason.DISCARD) }work?.cancel()active = false+ isFinishingSlow = falsestate = VoiceInputState.Idlesnapshot = nullsession = null@@ -467,20 +471,20 @@ private class LiveMicrophone(scope.launch(Dispatchers.IO) {val buffer = ByteBuffer.allocateDirect(3200).order(ByteOrder.LITTLE_ENDIAN)try {- while (!stopped.get()) {- buffer.clear()- val read = active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)- if (stopped.get() && read <= 0) break- if (- read <= 0 ||- audioManager.isMicrophoneMute ||+ captureMicrophoneAudio(+ isStopped = stopped::get,+ read = {+ buffer.clear()+ active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)+ },+ isInterrupted = {+ audioManager.isMicrophoneMute ||(android.os.Build.VERSION.SDK_INT >= 29 &&active.activeRecordingConfiguration?.isClientSilenced ==true) ||(routedDevice != null && active.routedDevice?.id != routedDevice)- ) {- throw IOException("Microphone interrupted")- }+ },+ ) { read ->routedDevice = active.routedDevice?.idbuffer.position(0)buffer.limit(read)@@ -495,7 +499,7 @@ private class LiveMicrophone(.toFloat().coerceIn(0f, 1f))- stream.feed(buffer)+ stream.feed(buffer) == DictationFeedResult.ACCEPTED}} catch (_: Exception) {if (!stopped.get()) {@@ -513,10 +517,9 @@ private class LiveMicrophone(suspend fun stopAndDrain() {stop()readJob?.join()- if (readJob == null) {- recorder?.release()- recorder = null- }+ // A reader canceled before dispatch never enters its cleanup block.+ recorder?.release()+ recorder = null}fun stop() {diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.ktindex b85adeef5c..4974e98aa2 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt@@ -26,6 +26,7 @@ import androidx.compose.material3.CircularProgressIndicatorimport androidx.compose.material3.Iconimport androidx.compose.material3.IconButtonimport androidx.compose.material3.Text+import androidx.compose.material3.TextButtonimport androidx.compose.runtime.Composableimport androidx.compose.runtime.LaunchedEffectimport androidx.compose.runtime.getValue@@ -63,7 +64,8 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {is VoiceInputState.Downloading -> "Downloading"VoiceInputState.Preparing,VoiceInputState.Idle -> "Preparing"- VoiceInputState.Transcribing -> "Transcribing"+ VoiceInputState.Transcribing ->+ if (controller.isFinishingSlow) "Still transcribing" else "Transcribing"VoiceInputState.Recording ->if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"else "Listening"@@ -98,6 +100,9 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {if (state.isRecording)VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))else Spacer(Modifier.weight(1f))+ if (controller.active && !state.isRecording) {+ TextButton(onClick = controller::discard) { Text("Cancel") }+ }}}diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.ktnew file mode 100644index 0000000000..aeebb06bd9--- /dev/null+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt@@ -0,0 +1,25 @@+package io.ente.ensu.chat++import org.junit.Assert.assertEquals+import org.junit.Test++class MicrophoneCaptureTest {+ @Test+ fun stopBetweenReadAndRouteCheckDeliversTheFinalBufferOnce() {+ var stopped = false+ val delivered = mutableListOf<Int>()+ captureMicrophoneAudio(+ isStopped = { stopped },+ read = { 3200 },+ isInterrupted = {+ // AudioRecord.stop() clears the route after a successful read.+ stopped = true+ true+ },+ ) { count ->+ delivered.add(count)+ true+ }+ assertEquals(listOf(3200), delivered)+ }+}diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swiftindex 5ba2e8d49a..260fb7715c 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift@@ -9,7 +9,7 @@ struct VoiceInputControls: View {switch service.state {case .downloading: return "Downloading"case .preparing: return "Preparing"- case .transcribing: return "Transcribing"+ case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"case .error(let message):if message.contains("permission") { return "Microphone access" }@@ -36,6 +36,11 @@ struct VoiceInputControls: View {} else {Spacer(minLength: 0)}+ if service.active, !service.state.isRecording {+ Button("Cancel", action: service.cancel)+ .font(EnsuTypography.small)+ .foregroundStyle(EnsuColor.textPrimary)+ }}}}diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 029d4c1a4d..897e4b98e3 100644--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -32,6 +32,7 @@ final class VoiceTranscriptionService: ObservableObject {@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)@Published private(set) var notice: String?@Published private(set) var state: VoiceInputState = .idle+ @Published private(set) var isFinishingSlow = falsevar onActivityChanged: (@MainActor () -> Void)?var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?private let provider: LlmProvider@@ -42,7 +43,7 @@ final class VoiceTranscriptionService: ObservableObject {private var stream: DictationStream?private var recorder: LivePcmRecorder?private var task: Task<Void, Never>?- private var finishingWatchdog: Task<Void, Never>?+ private var finishingNoticeTask: Task<Void, Never>?private var attemptFinished = falseprivate var insertion: DictationInsertion?private var generation = UUID()@@ -101,6 +102,7 @@ final class VoiceTranscriptionService: ObservableObject {generation = UUID()self.insertion = insertionattemptFinished = false+ isFinishingSlow = falsesession = DictationSession(live: usesLiveDictation)snapshot = session?.snapshot()active = true@@ -168,8 +170,8 @@ final class VoiceTranscriptionService: ObservableObject {defer {self.hasActiveTasks = falseif self.generation == token {- self.finishingWatchdog?.cancel()- self.finishingWatchdog = nil+ self.finishingNoticeTask?.cancel()+ self.finishingNoticeTask = nilself.recorder?.stop()self.recorder = nilself.stream = nil@@ -290,6 +292,7 @@ final class VoiceTranscriptionService: ObservableObject {hasText ? "Voice input stopped." : notice ?? "No speech detected.")}active = false+ isFinishingSlow = falsesession = nilsnapshot = nilinsertion = nil@@ -301,19 +304,14 @@ final class VoiceTranscriptionService: ObservableObject {private func watchFinalization(_ owner: DictationSession, stream: DictationStream, token: UUID){guard token == generation, active, self.stream === stream, owner.snapshot().workerActive,- finishingWatchdog == nil+ finishingNoticeTask == nilelse { return }- finishingWatchdog = Task { @MainActor [weak self] in+ finishingNoticeTask = Task { @MainActor [weak self] indo { try await Task.sleep(for: .seconds(10)) } catch { return }guard let self, self.generation == token, self.active, self.stream === stream,owner.snapshot().workerActiveelse { return }- self.recorder?.stop()- try? stream.abandon(reason: .failure)- self.task?.cancel()- self.snapshot = owner.snapshot()- self.notice =- "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."+ self.isFinishingSlow = true}}@@ -338,6 +336,8 @@ final class VoiceTranscriptionService: ObservableObject {recorder?.stop()try? stream?.abandon(reason: .interruption)task?.cancel()+ finishingNoticeTask?.cancel()+ isFinishingSlow = falsenotice = messagestate = .transcribing}@@ -346,12 +346,13 @@ final class VoiceTranscriptionService: ObservableObject {awaitingPermission = falsestartOnForeground = falsegeneration = UUID()- finishingWatchdog?.cancel()- finishingWatchdog = nil+ finishingNoticeTask?.cancel()+ finishingNoticeTask = nilrecorder?.stop()try? stream?.abandon(reason: .discard)task?.cancel()active = false+ isFinishingSlow = falsestate = .idlesnapshot = nilsession = nil@@ -502,7 +503,10 @@ private final class LiveCaptureDelivery: @unchecked Sendable {.littleEndian}}- try stream.feed(pcmLe: pcm)+ if try stream.feed(pcmLe: pcm) == .closed {+ close()+ return+ }}} catch {lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rsindex ba3f5cb152..90683bce04 100644--- a/rust/bindings/uniffi/ensu/src/voice.rs+++ b/rust/bindings/uniffi/ensu/src/voice.rs@@ -145,6 +145,13 @@ pub struct DictationStream {stream: LiveTranscription,started: AtomicBool,}++#[derive(Debug, Clone, Copy, uniffi::Enum)]+pub enum DictationFeedResult {+ Accepted,+ Closed,+}+#[uniffi::export]impl DictationStream {#[uniffi::constructor]@@ -164,8 +171,11 @@ impl DictationStream {started: AtomicBool::new(false),}))}- pub fn feed(&self, pcm_le: &[u8]) -> Result<(), DictationError> {- self.stream.feed(pcm_le).map_err(Into::into)+ pub fn feed(&self, pcm_le: &[u8]) -> Result<DictationFeedResult, DictationError> {+ Ok(match self.stream.feed(pcm_le)? {+ voice::CaptureFeedResult::Accepted => DictationFeedResult::Accepted,+ voice::CaptureFeedResult::Closed => DictationFeedResult::Closed,+ })}pub fn stop(&self, reason: DictationStopReason) -> Result<(), DictationError> {self.session.lock().session.stop(reason.into())?;@@ -232,7 +242,7 @@ impl DictationStream {.session.lock().apply(event, || self.stream.finish_if_empty())?;- if immediate && !notified_stop && self.session.lock().session.stop_reason().is_some() {+ if !notified_stop && self.session.lock().session.stop_reason().is_some() {notified_stop = true;callback.on_capture_closed();}diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rsindex 33960e7154..7c281996dc 100644--- a/rust/crates/ensu/src/transcription/live.rs+++ b/rust/crates/ensu/src/transcription/live.rs@@ -6,7 +6,8 @@ use transcribe_rs::vad::Vad;use super::{StreamNormalizer, Transcriber};use crate::voice::{- AlignedWord, CaptureFormat, CaptureQueue, Result, RollingRecognizer, VoiceError,+ AlignedWord, CaptureFeedResult, CaptureFormat, CaptureQueue, Result, RollingRecognizer,+ VoiceError,};pub enum LiveEvent {@@ -53,7 +54,7 @@ impl LiveTranscription {})}- pub fn feed(&self, pcm: &[u8]) -> Result<()> {+ pub fn feed(&self, pcm: &[u8]) -> Result<CaptureFeedResult> {self.queue.push(pcm)}pub fn finish(&self) {diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rsindex cb8eb41377..69f7c3f243 100644--- a/rust/crates/ensu/src/transcription/recorded.rs+++ b/rust/crates/ensu/src/transcription/recorded.rs@@ -181,6 +181,62 @@ mod tests {assert_eq!(covered, 1001);}+ #[test]+ fn silence_stop_preserves_transcription_when_a_capture_buffer_arrives_late() {+ use super::super::DictationState;+ use crate::voice::{CaptureFeedResult, StopReason};++ let stream = LiveTranscription::new(CaptureFormat {+ sample_rate: 16000,+ channels: 1,+ })+ .unwrap();+ let mut state = DictationState::new(false);+ state.session.begin_capture().unwrap();+ let mut audio = [0, 32].repeat(512);+ audio.extend(vec![0; (3 * 16000_usize).div_ceil(512) * 512 * 2]);+ let mut chunks = audio.chunks(1024);+ let mut capture_closed = false;+ let mut late_buffer = false;+ stream+ .run_recorded_with(+ || {+ Ok(Box::new(EnergyVad::new(+ super::super::VAD_FRAME_SAMPLES,+ 0.01,+ )))+ },+ |pcm| {+ assert_eq!(pcm, audio);+ Ok("Complete recorded transcript".into())+ },+ &mut |event| {+ let idle = matches!(event, LiveEvent::Idle);+ if matches!(event, LiveEvent::CaptureClosed) {+ capture_closed = true;+ }+ let control = state.apply(event, || stream.finish_if_empty())?;+ if idle && state.session.stop_reason() == Some(StopReason::Silence) {+ // A native read can finish after closure, before its stop callback.+ assert!(!capture_closed);+ assert_eq!(stream.feed(&[0; 3200])?, CaptureFeedResult::Closed);+ late_buffer = true;+ } else if idle {+ stream.feed(chunks.next().unwrap())?;+ }+ Ok(control)+ },+ )+ .unwrap();+ state.session.worker_exited(true).unwrap();+ assert!(late_buffer);+ assert!(capture_closed);+ assert_eq!(state.session.stop_reason(), Some(StopReason::Silence));+ assert!(!state.session.incomplete());+ assert_eq!(state.text(), "Complete recorded transcript");+ assert_eq!(stream.submitted_samples(), (audio.len() / 2) as u64);+ }+#[test]fn cancellation_during_batch_decode_does_not_publish_text() {let stream = LiveTranscription::new(CaptureFormat {diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rsindex 4e1a0e5df0..8c88eda0db 100644--- a/rust/crates/ensu/src/transcription/session.rs+++ b/rust/crates/ensu/src/transcription/session.rs@@ -27,7 +27,7 @@ impl DictationState {pub fn hypothesis(&self) -> String {format!("{}{}",- voice::boundary_separator(&self.text, &self.hypothesis),+ voice::transcript_separator(&self.text, &self.hypothesis),self.hypothesis)}@@ -64,7 +64,7 @@ impl DictationState {self.last_commit_end = end;self.session.observe_recognition(!text.trim().is_empty())?;self.text- .push_str(voice::boundary_separator(&self.text, &text));+ .push_str(voice::transcript_separator(&self.text, &text));self.text.push_str(&text);self.hypothesis.clear();}@@ -163,4 +163,66 @@ mod tests {assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);}}++ #[test]+ fn recognized_word_boundaries_survive_preview_and_commit() {+ fn apply_update(state: &mut DictationState, update: voice::RecognitionUpdate) {+ if let Some((start, end, text)) = update.commit {+ event(state, LiveEvent::Commit { start, end, text });+ }+ event(state, LiveEvent::Hypothesis(update.hypothesis));+ event(state, LiveEvent::Coverage(update.covered_through));+ }++ for (words, expected) in [+ (+ vec!["It", "costs", "$20.", "today."],+ "It costs $20. today.",+ ),+ (+ vec!["The", "change", "is", "-5", "percent."],+ "The change is -5 percent.",+ ),+ (+ vec!["Say", "\"hello.\"", "Then", "leave."],+ "Say \"hello.\" Then leave.",+ ),+ ] {+ let mut state = DictationState::new(true);+ state.session.begin_capture().unwrap();+ let mut rolling = voice::RollingRecognizer::default();+ let aligned = || {+ words+ .iter()+ .enumerate()+ .map(|(i, text)| voice::AlignedWord {+ text: (*text).into(),+ start: i as u64 * 1000,+ end: (i as u64 + 1) * 1000,+ })+ .collect()+ };+ for i in 0..3 {+ rolling.feed(&vec![0.2; 16000], true).unwrap();+ event(+ &mut state,+ LiveEvent::Audio {+ start: i * 16000,+ end: (i + 1) * 16000,+ voiced: true,+ },+ );+ let request = rolling.next_window().unwrap().unwrap();+ apply_update(&mut state, rolling.complete(&request, aligned()).unwrap());+ assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);+ }+ assert!(!state.text().is_empty());+ assert!(!state.hypothesis().is_empty());+ rolling.finish();+ let request = rolling.next_window().unwrap().unwrap();+ apply_update(&mut state, rolling.complete(&request, aligned()).unwrap());+ assert_eq!(state.text(), expected);+ assert!(state.hypothesis().is_empty());+ }+ }}diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rsindex 544e8941a3..675782eb50 100644--- a/rust/crates/ensu/src/voice/capture.rs+++ b/rust/crates/ensu/src/voice/capture.rs@@ -4,6 +4,12 @@ use std::sync::{Mutex, MutexGuard, PoisonError};use super::{Result, VoiceError};+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum CaptureFeedResult {+ Accepted,+ Closed,+}+#[derive(Clone, Copy, Debug)]pub struct CaptureFormat {pub sample_rate: u32,@@ -53,7 +59,7 @@ impl CaptureQueue {self.format}- pub fn push(&self, pcm: &[u8]) -> Result<()> {+ pub fn push(&self, pcm: &[u8]) -> Result<CaptureFeedResult> {if self.cancelled.load(Ordering::Acquire) {return Err(VoiceError::Cancelled);}@@ -71,7 +77,7 @@ impl CaptureQueue {let owned = pcm.to_vec();let mut pending = self.lock();if pending.closed {- return Err(VoiceError::InvalidState);+ return Ok(CaptureFeedResult::Closed);}self.submitted_frames.fetch_add((pcm.len() / frame_bytes) as u64, Ordering::AcqRel);@@ -83,7 +89,7 @@ impl CaptureQueue {}pending.bytes += pcm.len();pending.chunks.push_back(owned);- Ok(())+ Ok(CaptureFeedResult::Accepted)}pub fn take(&self) -> Option<Vec<u8>> {@@ -178,7 +184,9 @@ mod tests {assert_eq!(queue.take(), Some(vec![1, 0]));assert!(queue.finish_if_empty());assert!(queue.is_drained());- assert_eq!(queue.push(&[1, 0]), Err(VoiceError::InvalidState));+ assert_eq!(queue.push(&[1, 0]), Ok(CaptureFeedResult::Closed));+ assert_eq!(queue.submitted_frames(), 1);+ assert!(!queue.failed());queue.cancel();assert_eq!(queue.push(&[1, 0]), Err(VoiceError::Cancelled));}diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rsindex 983d956d39..8f22dadbf5 100644--- a/rust/crates/ensu/src/voice/composer.rs+++ b/rust/crates/ensu/src/voice/composer.rs@@ -25,28 +25,24 @@ pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {Err(VoiceError::InvalidTextRange)}-pub fn boundary_separator(left: &str, right: &str) -> &'static str {- fn unspaced(c: char) -> bool {- matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)- }- let left_edge = left- .chars()+fn left_edge(text: &str) -> Option<char> {+ text.chars().rev().find(|c| {!regex::regex!(r"\A[\p{M}\p{Join_Control}]\z").is_match(c.encode_utf8(&mut [0; 4]))})- .or_else(|| left.chars().next());- let right_edge = right.chars().next();- match (left_edge, right_edge) {+ .or_else(|| text.chars().next())+}++// Recognition already supplies word boundaries. Symbols and punctuation belong to+// those words and must not suppress their separators, including across commits.+pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {+ fn unspaced(c: char) -> bool {+ matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)+ }+ match (left_edge(left), right.chars().next()) {(Some(a), Some(b))- if (a.is_alphanumeric()- || matches!(- a,- '.' | '!' | '?' | ':' | ';' | ',' | '।' | '॥' | '؟' | '؛' | '،'- ))- && b.is_alphanumeric()- && !unspaced(a)- && !unspaced(b) =>+ if !a.is_whitespace() && !b.is_whitespace() && !unspaced(a) && !unspaced(b) =>{" "}@@ -54,6 +50,90 @@ pub fn boundary_separator(left: &str, right: &str) -> &'static str {}}+pub fn boundary_separator(left: &str, right: &str) -> &'static str {+ fn opening(c: char) -> bool {+ matches!(c, '¿' | '¡')+ || regex::regex!(r"\A[\p{Ps}\p{Pi}]\z").is_match(c.encode_utf8(&mut [0; 4]))+ }+ fn closing(c: char) -> bool {+ matches!(+ c,+ '.' | ','+ | '!'+ | '?'+ | ':'+ | ';'+ | '…'+ | '。'+ | ','+ | '!'+ | '?'+ | ':'+ | ';'+ | '、'+ | '।'+ | '॥'+ | '؟'+ | '؛'+ | '،'+ | '%'+ | '‰'+ | '‱'+ | '٪'+ | '؉'+ | '؊'+ | '°'+ ) || regex::regex!(r"\A[\p{Pe}\p{Pf}]\z").is_match(c.encode_utf8(&mut [0; 4]))+ }+ fn emoji(c: char) -> bool {+ regex::regex!(r"\A\p{Extended_Pictographic}\z").is_match(c.encode_utf8(&mut [0; 4]))+ }++ let separator = transcript_separator(left, right);+ let (Some(a), Some(b)) = (left_edge(left), right.chars().next()) else {+ return separator;+ };+ if separator.is_empty()+ || opening(a)+ || closing(b)+ || emoji(a)+ || emoji(b)+ || matches!(a, '#' | '@' | '_' | '/' | '\\' | '-' | '+' | '−')+ {+ return "";+ }++ // A currency prefix attaches to its amount; a trailing currency symbol is+ // already part of a completed amount and needs a word separator.+ if regex::regex!(r"\A\p{Sc}\z").is_match(a.encode_utf8(&mut [0; 4]))+ && !left.rsplit_once(a).is_some_and(|(before, _)| {+ before+ .trim_end()+ .chars()+ .next_back()+ .is_some_and(char::is_numeric)+ })+ {+ return "";+ }+ if matches!(a, '\'' | '"')+ && left_edge(left.trim_end_matches(['\'', '"']))+ .is_none_or(|c| c.is_whitespace() || opening(c))+ {+ return "";+ }+ if matches!(b, '\'' | '"')+ && right+ .trim_start_matches(['\'', '"'])+ .chars()+ .next()+ .is_none_or(|c| c.is_whitespace() || closing(c))+ {+ return "";+ }+ separator+}+#[derive(Debug, Clone, PartialEq, Eq)]pub struct ComposerAnchor {pub destination: String,@@ -261,5 +341,50 @@ mod tests {assert_eq!(boundary_separator("है।", "बैठक"), " ");assert_eq!(boundary_separator("نعم؟", "موعد"), " ");assert_eq!(boundary_separator("안녕", "친구"), " ");+ for (draft, dictated, expected) in [+ ("Charge 20%", "more tomorrow", "Charge 20% more tomorrow"),+ ("Charge 20‰", "extra", "Charge 20‰ extra"),+ ("Rotate 90°", "clockwise", "Rotate 90° clockwise"),+ ("Budget 20€", "today", "Budget 20€ today"),+ ("Budget 20 €", "today", "Budget 20 € today"),+ ("نسبة 20٪", "زيادة", "نسبة 20٪ زيادة"),+ ("Notes (draft)", "hello", "Notes (draft) hello"),+ ("Budget", "$20.", "Budget $20."),+ ("Budget €", "20", "Budget €20"),+ ("Budget -", "20", "Budget -20"),+ ("Change:", "-5 percent.", "Change: -5 percent."),+ ("Say", "\"hello.\"", "Say \"hello.\""),+ (+ "Say \"hello.\"",+ "Then leave.",+ "Say \"hello.\" Then leave.",+ ),+ ("Say \"", "hello", "Say \"hello"),+ ("Notes (", "draft", "Notes (draft"),+ ("Notes (\"", "draft", "Notes (\"draft"),+ ("hello", ", world", "hello, world"),+ ("Notes (draft", ")", "Notes (draft)"),+ ("Say \"hello", "\"", "Say \"hello\""),+ ] {+ let end = draft.encode_utf16().count() as u32;+ let insert = insertion(end, end, dictated);+ let (preview, caret) = preview_insertion(draft, &insert.anchor, dictated).unwrap();+ assert_eq!(preview, expected);+ let mut text = draft.to_owned();+ let mut revision = 7;+ assert_eq!(+ ComposerTransfer::default()+ .apply(&insert, "chat", &mut revision, &mut text, true, false)+ .unwrap(),+ caret+ );+ assert_eq!(text, expected);+ // The same boundary can occur after the inserted text, too.+ let anchor = insertion(0, 0, draft).anchor;+ assert_eq!(+ preview_insertion(dictated, &anchor, draft).unwrap().0,+ expected+ );+ }}}diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rsindex ba43201173..cbd9e9050e 100644--- a/rust/crates/ensu/src/voice/rolling.rs+++ b/rust/crates/ensu/src/voice/rolling.rs@@ -1,7 +1,7 @@use std::collections::VecDeque;use super::{- AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, boundary_separator,+ AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, transcript_separator,};const PRE_SPEECH: u64 = VOICE_SAMPLE_RATE;@@ -299,7 +299,7 @@ fn alignment_key(text: &str) -> String {fn join_words(words: &[AlignedWord]) -> String {let mut text = String::new();for word in words {- text.push_str(boundary_separator(&text, &word.text));+ text.push_str(transcript_separator(&text, &word.text));text.push_str(&word.text);}text