Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Preserve dictation audio tails and transcript spacing

setalp1aa356015dd4
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt
new file mode 100644
index 0000000000..19b5e4ebce
--- /dev/null
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/MicrophoneCapture.kt
@@ -0,0 +1,21 @@
+package io.ente.ensu.chat
+
+import java.io.IOException
+
+internal fun captureMicrophoneAudio(
+ isStopped: () -> Boolean,
+ read: () -> Int,
+ isInterrupted: () -> Boolean,
+ deliver: (Int) -> Boolean,
+) {
+ while (!isStopped()) {
+ val count = read()
+ if (count <= 0) {
+ if (isStopped()) return
+ throw IOException("Microphone read failed")
+ }
+ // Stop changes the route too. Preserve a successful final read while draining.
+ if (isInterrupted() && !isStopped()) throw IOException("Microphone interrupted")
+ if (!deliver(count)) return
+ }
+}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index 52b58d1407..a0fcd2ee21 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -12,6 +12,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import io.ente.ensu.assets.AssetStore
import io.ente.ensu.bindings.DictationException
+import io.ente.ensu.bindings.DictationFeedResult
import io.ente.ensu.bindings.DictationInsertion
import io.ente.ensu.bindings.DictationPhase
import io.ente.ensu.bindings.DictationSession
@@ -94,7 +95,7 @@ internal class VoiceTranscriptionController(
private var stream: DictationStream? = null
private var capture: LiveMicrophone? = null
private var work: Job? = null
- private var finishingWatchdog: Job? = null
+ private var finishingNoticeJob: Job? = null
private var attemptFinished = false
private var insertion: DictationInsertion? = null
private var foreground = true
@@ -112,6 +113,9 @@ internal class VoiceTranscriptionController(
var state: VoiceInputState by mutableStateOf(VoiceInputState.Idle)
private set
+ var isFinishingSlow by mutableStateOf(false)
+ private set
+
var snapshot: DictationSnapshot? by mutableStateOf(null)
private set
@@ -133,6 +137,7 @@ internal class VoiceTranscriptionController(
generation++
initialRecordPending = true
attemptFinished = false
+ isFinishingSlow = false
this.insertion = insertion
session = DictationSession(live = usesLiveDictation)
snapshot = session?.snapshot()
@@ -289,8 +294,8 @@ internal class VoiceTranscriptionController(
withContext(NonCancellable) { ownedCapture?.stopAndDrain() }
hasActiveWork = false
if (token == generation) {
- finishingWatchdog?.cancel()
- finishingWatchdog = null
+ finishingNoticeJob?.cancel()
+ finishingNoticeJob = null
capture = null
stream = null
attemptFinished = true
@@ -319,6 +324,7 @@ internal class VoiceTranscriptionController(
else -> VoiceInputState.Error("Voice input stopped.")
}
active = false
+ isFinishingSlow = false
snapshot = null
session = null
insertion = null
@@ -336,10 +342,10 @@ internal class VoiceTranscriptionController(
!active ||
stream !== finishingStream ||
!owner.snapshot().workerActive ||
- finishingWatchdog != null
+ finishingNoticeJob != null
)
return
- finishingWatchdog = scope.launch {
+ finishingNoticeJob = scope.launch {
delay(10_000)
if (
token != generation ||
@@ -348,12 +354,7 @@ internal class VoiceTranscriptionController(
!owner.snapshot().workerActive
)
return@launch
- capture?.stop()
- runCatching { finishingStream.abandon(DictationStopReason.FAILURE) }
- work?.cancel()
- snapshot = owner.snapshot()
- notice =
- "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."
+ isFinishingSlow = true
}
}
@@ -382,6 +383,8 @@ internal class VoiceTranscriptionController(
capture?.stop()
runCatching { stream?.abandon(DictationStopReason.INTERRUPTION) }
work?.cancel()
+ finishingNoticeJob?.cancel()
+ isFinishingSlow = false
notice = message
state = VoiceInputState.Transcribing
}
@@ -390,12 +393,13 @@ internal class VoiceTranscriptionController(
awaitingPermission = false
startOnForeground = false
generation++
- finishingWatchdog?.cancel()
- finishingWatchdog = null
+ finishingNoticeJob?.cancel()
+ finishingNoticeJob = null
capture?.stop()
runCatching { stream?.abandon(DictationStopReason.DISCARD) }
work?.cancel()
active = false
+ isFinishingSlow = false
state = VoiceInputState.Idle
snapshot = null
session = null
@@ -467,20 +471,20 @@ private class LiveMicrophone(
scope.launch(Dispatchers.IO) {
val buffer = ByteBuffer.allocateDirect(3200).order(ByteOrder.LITTLE_ENDIAN)
try {
- while (!stopped.get()) {
- buffer.clear()
- val read = active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)
- if (stopped.get() && read <= 0) break
- if (
- read <= 0 ||
- audioManager.isMicrophoneMute ||
+ captureMicrophoneAudio(
+ isStopped = stopped::get,
+ read = {
+ buffer.clear()
+ active.read(buffer, buffer.capacity(), AudioRecord.READ_BLOCKING)
+ },
+ isInterrupted = {
+ audioManager.isMicrophoneMute ||
(android.os.Build.VERSION.SDK_INT >= 29 &&
active.activeRecordingConfiguration?.isClientSilenced ==
true) ||
(routedDevice != null && active.routedDevice?.id != routedDevice)
- ) {
- throw IOException("Microphone interrupted")
- }
+ },
+ ) { read ->
routedDevice = active.routedDevice?.id
buffer.position(0)
buffer.limit(read)
@@ -495,7 +499,7 @@ private class LiveMicrophone(
.toFloat()
.coerceIn(0f, 1f)
)
- stream.feed(buffer)
+ stream.feed(buffer) == DictationFeedResult.ACCEPTED
}
} catch (_: Exception) {
if (!stopped.get()) {
@@ -513,10 +517,9 @@ private class LiveMicrophone(
suspend fun stopAndDrain() {
stop()
readJob?.join()
- if (readJob == null) {
- recorder?.release()
- recorder = null
- }
+ // A reader canceled before dispatch never enters its cleanup block.
+ recorder?.release()
+ recorder = null
}
fun stop() {
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
index b85adeef5c..4974e98aa2 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInputControls.kt
@@ -26,6 +26,7 @@ import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.Text
+import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
@@ -63,7 +64,8 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
is VoiceInputState.Downloading -> "Downloading"
VoiceInputState.Preparing,
VoiceInputState.Idle -> "Preparing"
- VoiceInputState.Transcribing -> "Transcribing"
+ VoiceInputState.Transcribing ->
+ if (controller.isFinishingSlow) "Still transcribing" else "Transcribing"
VoiceInputState.Recording ->
if (controller.snapshot?.phase == DictationPhase.COUNTDOWN) "Finishing"
else "Listening"
@@ -98,6 +100,9 @@ internal fun VoiceInputControls(controller: VoiceTranscriptionController) {
if (state.isRecording)
VoiceWaveform(controller.audioLevels, Modifier.weight(1f).height(24.dp))
else Spacer(Modifier.weight(1f))
+ if (controller.active && !state.isRecording) {
+ TextButton(onClick = controller::discard) { Text("Cancel") }
+ }
}
}
diff --git a/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt
new file mode 100644
index 0000000000..aeebb06bd9
--- /dev/null
+++ b/android/apps/ensu/app/src/test/java/io/ente/ensu/chat/MicrophoneCaptureTest.kt
@@ -0,0 +1,25 @@
+package io.ente.ensu.chat
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class MicrophoneCaptureTest {
+ @Test
+ fun stopBetweenReadAndRouteCheckDeliversTheFinalBufferOnce() {
+ var stopped = false
+ val delivered = mutableListOf<Int>()
+ captureMicrophoneAudio(
+ isStopped = { stopped },
+ read = { 3200 },
+ isInterrupted = {
+ // AudioRecord.stop() clears the route after a successful read.
+ stopped = true
+ true
+ },
+ ) { count ->
+ delivered.add(count)
+ true
+ }
+ assertEquals(listOf(3200), delivered)
+ }
+}
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
index 5ba2e8d49a..260fb7715c 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceInputControls.swift
@@ -9,7 +9,7 @@ struct VoiceInputControls: View {
switch service.state {
case .downloading: return "Downloading"
case .preparing: return "Preparing"
- case .transcribing: return "Transcribing"
+ case .transcribing: return service.isFinishingSlow ? "Still transcribing" : "Transcribing"
case .recording: return service.snapshot?.phase == .countdown ? "Finishing" : "Listening"
case .error(let message):
if message.contains("permission") { return "Microphone access" }
@@ -36,6 +36,11 @@ struct VoiceInputControls: View {
} else {
Spacer(minLength: 0)
}
+ if service.active, !service.state.isRecording {
+ Button("Cancel", action: service.cancel)
+ .font(EnsuTypography.small)
+ .foregroundStyle(EnsuColor.textPrimary)
+ }
}
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 029d4c1a4d..897e4b98e3 100644
--- a/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -32,6 +32,7 @@ final class VoiceTranscriptionService: ObservableObject {
@Published private(set) var audioLevels = Array(repeating: Float.zero, count: 24)
@Published private(set) var notice: String?
@Published private(set) var state: VoiceInputState = .idle
+ @Published private(set) var isFinishingSlow = false
var onActivityChanged: (@MainActor () -> Void)?
var onInsert: (@MainActor (DictationInsertion, String) -> Bool)?
private let provider: LlmProvider
@@ -42,7 +43,7 @@ final class VoiceTranscriptionService: ObservableObject {
private var stream: DictationStream?
private var recorder: LivePcmRecorder?
private var task: Task<Void, Never>?
- private var finishingWatchdog: Task<Void, Never>?
+ private var finishingNoticeTask: Task<Void, Never>?
private var attemptFinished = false
private var insertion: DictationInsertion?
private var generation = UUID()
@@ -101,6 +102,7 @@ final class VoiceTranscriptionService: ObservableObject {
generation = UUID()
self.insertion = insertion
attemptFinished = false
+ isFinishingSlow = false
session = DictationSession(live: usesLiveDictation)
snapshot = session?.snapshot()
active = true
@@ -168,8 +170,8 @@ final class VoiceTranscriptionService: ObservableObject {
defer {
self.hasActiveTasks = false
if self.generation == token {
- self.finishingWatchdog?.cancel()
- self.finishingWatchdog = nil
+ self.finishingNoticeTask?.cancel()
+ self.finishingNoticeTask = nil
self.recorder?.stop()
self.recorder = nil
self.stream = nil
@@ -290,6 +292,7 @@ final class VoiceTranscriptionService: ObservableObject {
hasText ? "Voice input stopped." : notice ?? "No speech detected.")
}
active = false
+ isFinishingSlow = false
session = nil
snapshot = nil
insertion = nil
@@ -301,19 +304,14 @@ final class VoiceTranscriptionService: ObservableObject {
private func watchFinalization(_ owner: DictationSession, stream: DictationStream, token: UUID)
{
guard token == generation, active, self.stream === stream, owner.snapshot().workerActive,
- finishingWatchdog == nil
+ finishingNoticeTask == nil
else { return }
- finishingWatchdog = Task { @MainActor [weak self] in
+ finishingNoticeTask = Task { @MainActor [weak self] in
do { try await Task.sleep(for: .seconds(10)) } catch { return }
guard let self, self.generation == token, self.active, self.stream === stream,
owner.snapshot().workerActive
else { return }
- self.recorder?.stop()
- try? stream.abandon(reason: .failure)
- self.task?.cancel()
- self.snapshot = owner.snapshot()
- self.notice =
- "Finishing took too long. Some speech may be missing. Review the available text while recognition stops."
+ self.isFinishingSlow = true
}
}
@@ -338,6 +336,8 @@ final class VoiceTranscriptionService: ObservableObject {
recorder?.stop()
try? stream?.abandon(reason: .interruption)
task?.cancel()
+ finishingNoticeTask?.cancel()
+ isFinishingSlow = false
notice = message
state = .transcribing
}
@@ -346,12 +346,13 @@ final class VoiceTranscriptionService: ObservableObject {
awaitingPermission = false
startOnForeground = false
generation = UUID()
- finishingWatchdog?.cancel()
- finishingWatchdog = nil
+ finishingNoticeTask?.cancel()
+ finishingNoticeTask = nil
recorder?.stop()
try? stream?.abandon(reason: .discard)
task?.cancel()
active = false
+ isFinishingSlow = false
state = .idle
snapshot = nil
session = nil
@@ -502,7 +503,10 @@ private final class LiveCaptureDelivery: @unchecked Sendable {
.littleEndian
}
}
- try stream.feed(pcmLe: pcm)
+ if try stream.feed(pcmLe: pcm) == .closed {
+ close()
+ return
+ }
}
} catch {
lock.lock(); let wasAccepting = accepting; accepting = false; lock.unlock()
diff --git a/rust/bindings/uniffi/ensu/src/voice.rs b/rust/bindings/uniffi/ensu/src/voice.rs
index ba3f5cb152..90683bce04 100644
--- a/rust/bindings/uniffi/ensu/src/voice.rs
+++ b/rust/bindings/uniffi/ensu/src/voice.rs
@@ -145,6 +145,13 @@ pub struct DictationStream {
stream: LiveTranscription,
started: AtomicBool,
}
+
+#[derive(Debug, Clone, Copy, uniffi::Enum)]
+pub enum DictationFeedResult {
+ Accepted,
+ Closed,
+}
+
#[uniffi::export]
impl DictationStream {
#[uniffi::constructor]
@@ -164,8 +171,11 @@ impl DictationStream {
started: AtomicBool::new(false),
}))
}
- pub fn feed(&self, pcm_le: &[u8]) -> Result<(), DictationError> {
- self.stream.feed(pcm_le).map_err(Into::into)
+ pub fn feed(&self, pcm_le: &[u8]) -> Result<DictationFeedResult, DictationError> {
+ Ok(match self.stream.feed(pcm_le)? {
+ voice::CaptureFeedResult::Accepted => DictationFeedResult::Accepted,
+ voice::CaptureFeedResult::Closed => DictationFeedResult::Closed,
+ })
}
pub fn stop(&self, reason: DictationStopReason) -> Result<(), DictationError> {
self.session.lock().session.stop(reason.into())?;
@@ -232,7 +242,7 @@ impl DictationStream {
.session
.lock()
.apply(event, || self.stream.finish_if_empty())?;
- if immediate && !notified_stop && self.session.lock().session.stop_reason().is_some() {
+ if !notified_stop && self.session.lock().session.stop_reason().is_some() {
notified_stop = true;
callback.on_capture_closed();
}
diff --git a/rust/crates/ensu/src/transcription/live.rs b/rust/crates/ensu/src/transcription/live.rs
index 33960e7154..7c281996dc 100644
--- a/rust/crates/ensu/src/transcription/live.rs
+++ b/rust/crates/ensu/src/transcription/live.rs
@@ -6,7 +6,8 @@ use transcribe_rs::vad::Vad;
use super::{StreamNormalizer, Transcriber};
use crate::voice::{
- AlignedWord, CaptureFormat, CaptureQueue, Result, RollingRecognizer, VoiceError,
+ AlignedWord, CaptureFeedResult, CaptureFormat, CaptureQueue, Result, RollingRecognizer,
+ VoiceError,
};
pub enum LiveEvent {
@@ -53,7 +54,7 @@ impl LiveTranscription {
})
}
- pub fn feed(&self, pcm: &[u8]) -> Result<()> {
+ pub fn feed(&self, pcm: &[u8]) -> Result<CaptureFeedResult> {
self.queue.push(pcm)
}
pub fn finish(&self) {
diff --git a/rust/crates/ensu/src/transcription/recorded.rs b/rust/crates/ensu/src/transcription/recorded.rs
index cb8eb41377..69f7c3f243 100644
--- a/rust/crates/ensu/src/transcription/recorded.rs
+++ b/rust/crates/ensu/src/transcription/recorded.rs
@@ -181,6 +181,62 @@ mod tests {
assert_eq!(covered, 1001);
}
+ #[test]
+ fn silence_stop_preserves_transcription_when_a_capture_buffer_arrives_late() {
+ use super::super::DictationState;
+ use crate::voice::{CaptureFeedResult, StopReason};
+
+ let stream = LiveTranscription::new(CaptureFormat {
+ sample_rate: 16000,
+ channels: 1,
+ })
+ .unwrap();
+ let mut state = DictationState::new(false);
+ state.session.begin_capture().unwrap();
+ let mut audio = [0, 32].repeat(512);
+ audio.extend(vec![0; (3 * 16000_usize).div_ceil(512) * 512 * 2]);
+ let mut chunks = audio.chunks(1024);
+ let mut capture_closed = false;
+ let mut late_buffer = false;
+ stream
+ .run_recorded_with(
+ || {
+ Ok(Box::new(EnergyVad::new(
+ super::super::VAD_FRAME_SAMPLES,
+ 0.01,
+ )))
+ },
+ |pcm| {
+ assert_eq!(pcm, audio);
+ Ok("Complete recorded transcript".into())
+ },
+ &mut |event| {
+ let idle = matches!(event, LiveEvent::Idle);
+ if matches!(event, LiveEvent::CaptureClosed) {
+ capture_closed = true;
+ }
+ let control = state.apply(event, || stream.finish_if_empty())?;
+ if idle && state.session.stop_reason() == Some(StopReason::Silence) {
+ // A native read can finish after closure, before its stop callback.
+ assert!(!capture_closed);
+ assert_eq!(stream.feed(&[0; 3200])?, CaptureFeedResult::Closed);
+ late_buffer = true;
+ } else if idle {
+ stream.feed(chunks.next().unwrap())?;
+ }
+ Ok(control)
+ },
+ )
+ .unwrap();
+ state.session.worker_exited(true).unwrap();
+ assert!(late_buffer);
+ assert!(capture_closed);
+ assert_eq!(state.session.stop_reason(), Some(StopReason::Silence));
+ assert!(!state.session.incomplete());
+ assert_eq!(state.text(), "Complete recorded transcript");
+ assert_eq!(stream.submitted_samples(), (audio.len() / 2) as u64);
+ }
+
#[test]
fn cancellation_during_batch_decode_does_not_publish_text() {
let stream = LiveTranscription::new(CaptureFormat {
diff --git a/rust/crates/ensu/src/transcription/session.rs b/rust/crates/ensu/src/transcription/session.rs
index 4e1a0e5df0..8c88eda0db 100644
--- a/rust/crates/ensu/src/transcription/session.rs
+++ b/rust/crates/ensu/src/transcription/session.rs
@@ -27,7 +27,7 @@ impl DictationState {
pub fn hypothesis(&self) -> String {
format!(
"{}{}",
- voice::boundary_separator(&self.text, &self.hypothesis),
+ voice::transcript_separator(&self.text, &self.hypothesis),
self.hypothesis
)
}
@@ -64,7 +64,7 @@ impl DictationState {
self.last_commit_end = end;
self.session.observe_recognition(!text.trim().is_empty())?;
self.text
- .push_str(voice::boundary_separator(&self.text, &text));
+ .push_str(voice::transcript_separator(&self.text, &text));
self.text.push_str(&text);
self.hypothesis.clear();
}
@@ -163,4 +163,66 @@ mod tests {
assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);
}
}
+
+ #[test]
+ fn recognized_word_boundaries_survive_preview_and_commit() {
+ fn apply_update(state: &mut DictationState, update: voice::RecognitionUpdate) {
+ if let Some((start, end, text)) = update.commit {
+ event(state, LiveEvent::Commit { start, end, text });
+ }
+ event(state, LiveEvent::Hypothesis(update.hypothesis));
+ event(state, LiveEvent::Coverage(update.covered_through));
+ }
+
+ for (words, expected) in [
+ (
+ vec!["It", "costs", "$20.", "today."],
+ "It costs $20. today.",
+ ),
+ (
+ vec!["The", "change", "is", "-5", "percent."],
+ "The change is -5 percent.",
+ ),
+ (
+ vec!["Say", "\"hello.\"", "Then", "leave."],
+ "Say \"hello.\" Then leave.",
+ ),
+ ] {
+ let mut state = DictationState::new(true);
+ state.session.begin_capture().unwrap();
+ let mut rolling = voice::RollingRecognizer::default();
+ let aligned = || {
+ words
+ .iter()
+ .enumerate()
+ .map(|(i, text)| voice::AlignedWord {
+ text: (*text).into(),
+ start: i as u64 * 1000,
+ end: (i as u64 + 1) * 1000,
+ })
+ .collect()
+ };
+ for i in 0..3 {
+ rolling.feed(&vec![0.2; 16000], true).unwrap();
+ event(
+ &mut state,
+ LiveEvent::Audio {
+ start: i * 16000,
+ end: (i + 1) * 16000,
+ voiced: true,
+ },
+ );
+ let request = rolling.next_window().unwrap().unwrap();
+ apply_update(&mut state, rolling.complete(&request, aligned()).unwrap());
+ assert_eq!(state.text().to_owned() + &state.hypothesis(), expected);
+ }
+ assert!(!state.text().is_empty());
+ assert!(!state.hypothesis().is_empty());
+ rolling.finish();
+ let request = rolling.next_window().unwrap().unwrap();
+ apply_update(&mut state, rolling.complete(&request, aligned()).unwrap());
+ assert_eq!(state.text(), expected);
+ assert!(state.hypothesis().is_empty());
+ }
+ }
}
diff --git a/rust/crates/ensu/src/voice/capture.rs b/rust/crates/ensu/src/voice/capture.rs
index 544e8941a3..675782eb50 100644
--- a/rust/crates/ensu/src/voice/capture.rs
+++ b/rust/crates/ensu/src/voice/capture.rs
@@ -4,6 +4,12 @@ use std::sync::{Mutex, MutexGuard, PoisonError};
use super::{Result, VoiceError};
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CaptureFeedResult {
+ Accepted,
+ Closed,
+}
+
#[derive(Clone, Copy, Debug)]
pub struct CaptureFormat {
pub sample_rate: u32,
@@ -53,7 +59,7 @@ impl CaptureQueue {
self.format
}
- pub fn push(&self, pcm: &[u8]) -> Result<()> {
+ pub fn push(&self, pcm: &[u8]) -> Result<CaptureFeedResult> {
if self.cancelled.load(Ordering::Acquire) {
return Err(VoiceError::Cancelled);
}
@@ -71,7 +77,7 @@ impl CaptureQueue {
let owned = pcm.to_vec();
let mut pending = self.lock();
if pending.closed {
- return Err(VoiceError::InvalidState);
+ return Ok(CaptureFeedResult::Closed);
}
self.submitted_frames
.fetch_add((pcm.len() / frame_bytes) as u64, Ordering::AcqRel);
@@ -83,7 +89,7 @@ impl CaptureQueue {
}
pending.bytes += pcm.len();
pending.chunks.push_back(owned);
- Ok(())
+ Ok(CaptureFeedResult::Accepted)
}
pub fn take(&self) -> Option<Vec<u8>> {
@@ -178,7 +184,9 @@ mod tests {
assert_eq!(queue.take(), Some(vec![1, 0]));
assert!(queue.finish_if_empty());
assert!(queue.is_drained());
- assert_eq!(queue.push(&[1, 0]), Err(VoiceError::InvalidState));
+ assert_eq!(queue.push(&[1, 0]), Ok(CaptureFeedResult::Closed));
+ assert_eq!(queue.submitted_frames(), 1);
+ assert!(!queue.failed());
queue.cancel();
assert_eq!(queue.push(&[1, 0]), Err(VoiceError::Cancelled));
}
diff --git a/rust/crates/ensu/src/voice/composer.rs b/rust/crates/ensu/src/voice/composer.rs
index 983d956d39..8f22dadbf5 100644
--- a/rust/crates/ensu/src/voice/composer.rs
+++ b/rust/crates/ensu/src/voice/composer.rs
@@ -25,28 +25,24 @@ pub fn utf16_range(text: &str, start: u32, end: u32) -> Result<Range<usize>> {
Err(VoiceError::InvalidTextRange)
}
-pub fn boundary_separator(left: &str, right: &str) -> &'static str {
- fn unspaced(c: char) -> bool {
- matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)
- }
- let left_edge = left
- .chars()
+fn left_edge(text: &str) -> Option<char> {
+ text.chars()
.rev()
.find(|c| {
!regex::regex!(r"\A[\p{M}\p{Join_Control}]\z").is_match(c.encode_utf8(&mut [0; 4]))
})
- .or_else(|| left.chars().next());
- let right_edge = right.chars().next();
- match (left_edge, right_edge) {
+ .or_else(|| text.chars().next())
+}
+
+// Recognition already supplies word boundaries. Symbols and punctuation belong to
+// those words and must not suppress their separators, including across commits.
+pub(crate) fn transcript_separator(left: &str, right: &str) -> &'static str {
+ fn unspaced(c: char) -> bool {
+ matches!(c as u32, 0x3040..=0x30ff | 0x3400..=0x9fff | 0x20000..=0x3134f)
+ }
+ match (left_edge(left), right.chars().next()) {
(Some(a), Some(b))
- if (a.is_alphanumeric()
- || matches!(
- a,
- '.' | '!' | '?' | ':' | ';' | ',' | '।' | '॥' | '؟' | '؛' | '،'
- ))
- && b.is_alphanumeric()
- && !unspaced(a)
- && !unspaced(b) =>
+ if !a.is_whitespace() && !b.is_whitespace() && !unspaced(a) && !unspaced(b) =>
{
" "
}
@@ -54,6 +50,90 @@ pub fn boundary_separator(left: &str, right: &str) -> &'static str {
}
}
+pub fn boundary_separator(left: &str, right: &str) -> &'static str {
+ fn opening(c: char) -> bool {
+ matches!(c, '¿' | '¡')
+ || regex::regex!(r"\A[\p{Ps}\p{Pi}]\z").is_match(c.encode_utf8(&mut [0; 4]))
+ }
+ fn closing(c: char) -> bool {
+ matches!(
+ c,
+ '.' | ','
+ | '!'
+ | '?'
+ | ':'
+ | ';'
+ | '…'
+ | '。'
+ | ','
+ | '!'
+ | '?'
+ | ':'
+ | ';'
+ | '、'
+ | '।'
+ | '॥'
+ | '؟'
+ | '؛'
+ | '،'
+ | '%'
+ | '‰'
+ | '‱'
+ | '٪'
+ | '؉'
+ | '؊'
+ | '°'
+ ) || regex::regex!(r"\A[\p{Pe}\p{Pf}]\z").is_match(c.encode_utf8(&mut [0; 4]))
+ }
+ fn emoji(c: char) -> bool {
+ regex::regex!(r"\A\p{Extended_Pictographic}\z").is_match(c.encode_utf8(&mut [0; 4]))
+ }
+
+ let separator = transcript_separator(left, right);
+ let (Some(a), Some(b)) = (left_edge(left), right.chars().next()) else {
+ return separator;
+ };
+ if separator.is_empty()
+ || opening(a)
+ || closing(b)
+ || emoji(a)
+ || emoji(b)
+ || matches!(a, '#' | '@' | '_' | '/' | '\\' | '-' | '+' | '−')
+ {
+ return "";
+ }
+
+ // A currency prefix attaches to its amount; a trailing currency symbol is
+ // already part of a completed amount and needs a word separator.
+ if regex::regex!(r"\A\p{Sc}\z").is_match(a.encode_utf8(&mut [0; 4]))
+ && !left.rsplit_once(a).is_some_and(|(before, _)| {
+ before
+ .trim_end()
+ .chars()
+ .next_back()
+ .is_some_and(char::is_numeric)
+ })
+ {
+ return "";
+ }
+ if matches!(a, '\'' | '"')
+ && left_edge(left.trim_end_matches(['\'', '"']))
+ .is_none_or(|c| c.is_whitespace() || opening(c))
+ {
+ return "";
+ }
+ if matches!(b, '\'' | '"')
+ && right
+ .trim_start_matches(['\'', '"'])
+ .chars()
+ .next()
+ .is_none_or(|c| c.is_whitespace() || closing(c))
+ {
+ return "";
+ }
+ separator
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ComposerAnchor {
pub destination: String,
@@ -261,5 +341,50 @@ mod tests {
assert_eq!(boundary_separator("है।", "बैठक"), " ");
assert_eq!(boundary_separator("نعم؟", "موعد"), " ");
assert_eq!(boundary_separator("안녕", "친구"), " ");
+ for (draft, dictated, expected) in [
+ ("Charge 20%", "more tomorrow", "Charge 20% more tomorrow"),
+ ("Charge 20‰", "extra", "Charge 20‰ extra"),
+ ("Rotate 90°", "clockwise", "Rotate 90° clockwise"),
+ ("Budget 20€", "today", "Budget 20€ today"),
+ ("Budget 20 €", "today", "Budget 20 € today"),
+ ("نسبة 20٪", "زيادة", "نسبة 20٪ زيادة"),
+ ("Notes (draft)", "hello", "Notes (draft) hello"),
+ ("Budget", "$20.", "Budget $20."),
+ ("Budget €", "20", "Budget €20"),
+ ("Budget -", "20", "Budget -20"),
+ ("Change:", "-5 percent.", "Change: -5 percent."),
+ ("Say", "\"hello.\"", "Say \"hello.\""),
+ (
+ "Say \"hello.\"",
+ "Then leave.",
+ "Say \"hello.\" Then leave.",
+ ),
+ ("Say \"", "hello", "Say \"hello"),
+ ("Notes (", "draft", "Notes (draft"),
+ ("Notes (\"", "draft", "Notes (\"draft"),
+ ("hello", ", world", "hello, world"),
+ ("Notes (draft", ")", "Notes (draft)"),
+ ("Say \"hello", "\"", "Say \"hello\""),
+ ] {
+ let end = draft.encode_utf16().count() as u32;
+ let insert = insertion(end, end, dictated);
+ let (preview, caret) = preview_insertion(draft, &insert.anchor, dictated).unwrap();
+ assert_eq!(preview, expected);
+ let mut text = draft.to_owned();
+ let mut revision = 7;
+ assert_eq!(
+ ComposerTransfer::default()
+ .apply(&insert, "chat", &mut revision, &mut text, true, false)
+ .unwrap(),
+ caret
+ );
+ assert_eq!(text, expected);
+ // The same boundary can occur after the inserted text, too.
+ let anchor = insertion(0, 0, draft).anchor;
+ assert_eq!(
+ preview_insertion(dictated, &anchor, draft).unwrap().0,
+ expected
+ );
+ }
}
}
diff --git a/rust/crates/ensu/src/voice/rolling.rs b/rust/crates/ensu/src/voice/rolling.rs
index ba43201173..cbd9e9050e 100644
--- a/rust/crates/ensu/src/voice/rolling.rs
+++ b/rust/crates/ensu/src/voice/rolling.rs
@@ -1,7 +1,7 @@
use std::collections::VecDeque;
use super::{
- AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, boundary_separator,
+ AudioRing, MAX_WINDOW_SAMPLES, Result, VOICE_SAMPLE_RATE, VoiceError, transcript_separator,
};
const PRE_SPEECH: u64 = VOICE_SAMPLE_RATE;
@@ -299,7 +299,7 @@ fn alignment_key(text: &str) -> String {
fn join_words(words: &[AlignedWord]) -> String {
let mut text = String::new();
for word in words {
- text.push_str(boundary_separator(&text, &word.text));
+ text.push_str(transcript_separator(&text, &word.text));
text.push_str(&word.text);
}
text