Skip to content
TypeScript

fix(mobile): transcribe Android audio during recording (#31300)

Alex66633fb6826c
diff --git a/apps/mobile/modules/t3-voice-transcription/android/build.gradle b/apps/mobile/modules/t3-voice-transcription/android/build.gradle
index 05995a7cc4c..ab2d08e0706 100644
--- a/apps/mobile/modules/t3-voice-transcription/android/build.gradle
+++ b/apps/mobile/modules/t3-voice-transcription/android/build.gradle
@@ -16,9 +16,11 @@ android {
dependencies {
implementation project(':expo-modules-core')
+ implementation project(':expo-audio')
implementation 'androidx.media3:media3-common:1.9.1'
implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
testImplementation 'junit:junit:4.13.2'
+ testImplementation 'com.facebook.react:react-android'
testImplementation 'org.robolectric:robolectric:4.16.1'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt
index 95423844c1d..6884d720dc6 100644
--- a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt
@@ -158,5 +158,16 @@ internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) :
}
}
+ suspend fun transcribeLive(capture: LivePcmCapture, pipe: PcmPipe): String = pipe.use {
+ val request = speechRecognizerRequest { audioSource = pipe.source }
+ try {
+ collectGoogleVoiceTranscript(
+ livePcmResponses(capture, pipe) { recognizer.startRecognition(request) }
+ )
+ } finally {
+ withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
+ }
+ }
+
override fun close() = recognizer.close()
}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LivePcmCapture.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LivePcmCapture.kt
new file mode 100644
index 00000000000..bd728283d8a
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LivePcmCapture.kt
@@ -0,0 +1,216 @@
+package expo.modules.t3voicetranscription
+
+import java.io.File
+import kotlin.math.abs
+import kotlin.math.log10
+import kotlin.time.TimeSource
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeout
+
+internal interface PcmCaptureSource {
+ val capacityBytes: Int
+ fun start()
+ suspend fun read(buffer: ByteArray, blocking: Boolean): Int
+ fun stop()
+ fun release()
+}
+
+internal class LivePcmCapture(
+ private val source: PcmCaptureSource,
+ val file: File,
+ private val output: PcmOutput,
+ private val clock: TimeSource = TimeSource.Monotonic
+) {
+ @Volatile private var worker: Job? = null
+ private val completion = CompletableDeferred<Result<Unit>>()
+ private val microphoneStopped = CompletableDeferred<Result<Unit>>()
+
+ @Volatile private var stopping = false
+ private var sourceStopped = false
+ private var sourceReleased = false
+
+ @Volatile var bytes = 0L
+ private set
+
+ @Volatile var metering = -160.0
+ private set
+ val isFinished get() = completion.isCompleted
+
+ @Volatile var inputExhausted = false
+ private set
+ val hasStarted get() = worker != null
+ val isRecording get() = hasStarted && !microphoneStopped.isCompleted
+
+ fun start(scope: CoroutineScope, seconds: Double, onFinished: (Throwable?) -> Unit) {
+ check(worker == null && !sourceReleased) { "The microphone is not prepared." }
+ worker = scope.launch {
+ var result = runCatching {
+ source.start()
+ writeCapture(seconds, onFinished)
+ requireCompleteAudio(bytes > 0 && file.length() == bytes) {
+ "The microphone did not finish its captured audio."
+ }
+ }
+ result = finishSource(result)
+ inputExhausted = result.isSuccess
+ result = runCatching { output.finish() }.fold(
+ onSuccess = { result },
+ onFailure = { failure -> mergeFailure(result, failure) }
+ )
+ if (microphoneStopped.complete(result)) onFinished(result.exceptionOrNull())
+ completion.complete(result)
+ }.also { job ->
+ job.invokeOnCompletion { error ->
+ if (!completion.isCompleted) {
+ val result =
+ finishSource(Result.failure(error ?: CancellationException("Capture stopped.")))
+ completion.complete(result)
+ microphoneStopped.complete(result)
+ runCatching { output.finish() }
+ }
+ }
+ }
+ }
+
+ private suspend fun writeCapture(
+ seconds: Double,
+ onFinished: (Throwable?) -> Unit
+ ) = coroutineScope {
+ val started = clock.markNow()
+ val chunks = Channel<ByteArray>(100)
+ var sent = 0L
+ val sender = launch {
+ var deadline = 0L
+ for (chunk in chunks) {
+ deadline = maxOf(
+ deadline + (chunk.size * 1000L + 31999) / 32000,
+ started.elapsedNow().inWholeMilliseconds
+ )
+ delay((deadline - started.elapsedNow().inWholeMilliseconds).coerceAtLeast(0))
+ output.write(chunk, chunk.size)
+ deadline = started.elapsedNow().inWholeMilliseconds
+ sent += chunk.size
+ }
+ }
+ val result = finishSource(runCatching { captureChunks(seconds, chunks) })
+ microphoneStopped.complete(result)
+ if (result.exceptionOrNull() !is CancellationException) onFinished(result.exceptionOrNull())
+ chunks.close()
+ result.getOrThrow()
+ sender.join()
+ requireCompleteAudio(sent == bytes) { "Speech recognition did not receive all captured audio." }
+ }
+
+ private suspend fun captureChunks(seconds: Double, chunks: Channel<ByteArray>) {
+ val buffer = ByteArray(3200)
+ var drained = 0
+ file.outputStream().use { stream ->
+ while (true) {
+ currentCoroutineContext().ensureActive()
+ val draining = stopping || bytes >= seconds * 32000
+ val count = source.read(buffer, !draining)
+ requireCompleteAudio(count in 0..buffer.size && count % 2 == 0) {
+ "The microphone returned incomplete audio."
+ }
+ if (count == 0) {
+ requireCompleteAudio(draining) { "The microphone stopped supplying audio." }
+ break
+ }
+ if (draining) {
+ drained += count
+ requireCompleteAudio(drained <= source.capacityBytes + buffer.size) {
+ "The microphone audio queue did not drain."
+ }
+ }
+ stream.write(buffer, 0, count)
+ // Keep draining the microphone while inference starts or applies backpressure.
+ if (!chunks.trySend(buffer.copyOf(count)).isSuccess) {
+ throw VoiceTranscriptionException(
+ "audio-overflow",
+ "Speech recognition fell behind capture."
+ )
+ }
+ bytes += count
+ var peak = 0
+ for (i in 0 until count step 2) {
+ val sample = ((buffer[i].toInt() and 255) or (buffer[i + 1].toInt() shl 8)).toShort()
+ peak = maxOf(peak, abs(sample.toInt()))
+ }
+ metering = if (peak == 0) -160.0 else 20 * log10(peak / 32768.0)
+ }
+ }
+ }
+
+ suspend fun stop() {
+ stopping = true
+ if (worker == null) {
+ finishSource(Result.success(Unit)).getOrThrow()
+ output.finish()
+ return
+ }
+ try {
+ withTimeout(2000) { microphoneStopped.await().getOrThrow() }
+ } catch (error: TimeoutCancellationException) {
+ cancel()
+ throw VoiceTranscriptionException("incomplete-audio", "The microphone did not stop.", error)
+ }
+ }
+
+ suspend fun awaitFinished() = completion.await().getOrThrow()
+
+ fun abort() {
+ stopping = true
+ worker?.cancel()
+ runCatching { stopSource() }
+ runCatching { output.finish() }
+ if (worker == null) {
+ val result = finishSource(Result.failure(CancellationException("Capture was cancelled.")))
+ completion.complete(result)
+ microphoneStopped.complete(result)
+ }
+ }
+
+ suspend fun cancel() = withContext(NonCancellable) {
+ abort()
+ worker?.join()
+ finishSource(Result.success(Unit))
+ }
+
+ @Synchronized private fun stopSource() {
+ if (sourceStopped) return
+ sourceStopped = true
+ source.stop()
+ }
+
+ @Synchronized private fun finishSource(result: Result<Unit>): Result<Unit> {
+ if (sourceReleased) return result
+ sourceReleased = true
+ var outcome = runCatching { stopSource() }.fold(
+ onSuccess = { result },
+ onFailure = { mergeFailure(result, it) }
+ )
+ outcome = runCatching { source.release() }.fold(
+ onSuccess = { outcome },
+ onFailure = { mergeFailure(outcome, it) }
+ )
+ return outcome
+ }
+}
+
+private fun mergeFailure(result: Result<Unit>, failure: Throwable): Result<Unit> {
+ val original = result.exceptionOrNull() ?: return Result.failure(failure)
+ if (original !== failure) original.addSuppressed(failure)
+ return result
+}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LiveVoiceRecorder.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LiveVoiceRecorder.kt
new file mode 100644
index 00000000000..b5032b587cb
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LiveVoiceRecorder.kt
@@ -0,0 +1,198 @@
+package expo.modules.t3voicetranscription
+
+import android.content.Context
+import android.media.AudioFormat
+import android.media.AudioManager
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.net.Uri
+import androidx.annotation.RequiresApi
+import expo.modules.audio.RecordingInputRoute
+import expo.modules.audio.getMapFromDeviceInfo
+import expo.modules.audio.recordingInputs
+import expo.modules.audio.recordingInputUid
+import expo.modules.kotlin.AppContext
+import expo.modules.kotlin.sharedobjects.SharedObject
+import java.io.File
+import java.util.UUID
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.job
+
+@RequiresApi(31)
+internal class LiveVoiceRecorder(
+ private val context: Context,
+ appContext: AppContext,
+ private val scope: CoroutineScope,
+ private val onRelease: (String) -> Unit
+) : SharedObject(appContext) {
+ val id = UUID.randomUUID().toString()
+ private val manager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
+ private val route = RecordingInputRoute(manager)
+ private var audio: AudioRecord? = null
+ private var pipe: PcmPipe? = null
+ private var capture: LivePcmCapture? = null
+ private var inputSelection: Job? = null
+ private var inputCanceled = false
+ private var inferenceStarted = false
+ val uri: String? get() = capture?.file?.let { Uri.fromFile(it).toString() }
+
+ @Suppress("TooGenericExceptionCaught") // Release native handles for every setup failure.
+ fun prepare() {
+ check(capture?.isRecording != true) { "Another voice recording is active." }
+ capture?.abort()
+ pipe?.close()
+ inputCanceled = false
+ inferenceStarted = false
+ val capacity = maxOf(
+ 32000,
+ AudioRecord.getMinBufferSize(
+ 16000,
+ AudioFormat.CHANNEL_IN_MONO,
+ AudioFormat.ENCODING_PCM_16BIT
+ )
+ )
+ val recorder = AudioRecord.Builder()
+ .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
+ .setAudioFormat(
+ AudioFormat.Builder().setSampleRate(16000)
+ .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
+ .setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()
+ )
+ .setBufferSizeInBytes(capacity)
+ .build()
+ try {
+ check(recorder.state == AudioRecord.STATE_INITIALIZED) { "The microphone could not prepare." }
+ val output = PcmPipe.open()
+ try {
+ val file = File.createTempFile("dictation-live-", ".pcm", context.cacheDir)
+ capture = LivePcmCapture(
+ object : PcmCaptureSource {
+ override val capacityBytes = capacity
+ override fun start() {
+ recorder.startRecording()
+ check(recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
+ "The microphone could not start."
+ }
+ }
+ override suspend fun read(buffer: ByteArray, blocking: Boolean) = recorder.read(
+ buffer,
+ 0,
+ buffer.size,
+ if (blocking) AudioRecord.READ_BLOCKING else AudioRecord.READ_NON_BLOCKING
+ )
+ override fun stop() {
+ if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) recorder.stop()
+ }
+ override fun release() {
+ try {
+ recorder.release()
+ } finally {
+ route.close()
+ }
+ }
+ },
+ file,
+ output
+ )
+ audio = recorder
+ pipe = output
+ } catch (error: Throwable) {
+ output.close()
+ throw error
+ }
+ } catch (error: Throwable) {
+ recorder.release()
+ throw error
+ }
+ }
+
+ fun record(seconds: Double) {
+ val recording = requireNotNull(capture) { "The microphone is not prepared." }
+ recording.start(scope, seconds) { error ->
+ emit(
+ "recordingStatusUpdate",
+ mapOf(
+ "id" to id,
+ "isFinished" to true,
+ "hasError" to (error != null),
+ "error" to error?.message,
+ "url" to Uri.fromFile(recording.file).toString()
+ )
+ )
+ }
+ }
+
+ suspend fun stop() {
+ val recording = capture
+ val output = pipe
+ try {
+ val selection = synchronized(this) { inputSelection }
+ selection?.cancel()
+ selection?.join()
+ recording?.stop()
+ } finally {
+ // Recognition can start after Stop and still needs the captured PCM pipe.
+ if (recording?.hasStarted != true) output?.close()
+ }
+ }
+
+ fun status() = mapOf(
+ "canRecord" to (capture != null),
+ "isRecording" to (capture?.isRecording == true),
+ "durationMillis" to (capture?.bytes?.div(32) ?: 0),
+ "fileSize" to (capture?.file?.length() ?: 0),
+ "metering" to (capture?.metering ?: -160.0),
+ "mediaServicesDidReset" to false,
+ "url" to uri
+ )
+
+ fun inputs() = recordingInputs(manager).let { inputs ->
+ inputs.map { getMapFromDeviceInfo(it, inputs) }
+ }
+
+ // Release the preferred route on cancellation as well as selection errors.
+ @Suppress("TooGenericExceptionCaught")
+ suspend fun setInput(uid: String) {
+ val job = currentCoroutineContext().job
+ synchronized(this) {
+ check(!inputCanceled) { "Microphone selection was cancelled." }
+ inputSelection = job
+ }
+ try {
+ val inputs = recordingInputs(manager)
+ val selected = inputs.firstOrNull { recordingInputUid(it, inputs) == uid }
+ ?: error("The preferred microphone is disconnected.")
+ route.select(selected)
+ val routed = recordingInputs(manager)
+ val source = routed.firstOrNull { it.isSource && recordingInputUid(it, routed) == uid }
+ ?: error("The preferred microphone is disconnected.")
+ check(audio?.setPreferredDevice(source) == true) { "Could not select the microphone." }
+ } catch (error: Throwable) {
+ route.close()
+ throw error
+ } finally {
+ synchronized(this) { if (inputSelection === job) inputSelection = null }
+ }
+ }
+
+ @Synchronized fun cancelInputSelection() {
+ inputCanceled = true
+ inputSelection?.cancel()
+ }
+
+ suspend fun transcribe(uri: String, client: MlKitSpeechClient): String {
+ check(this.uri == uri && !inferenceStarted) { "The voice recording is no longer available." }
+ inferenceStarted = true
+ return client.transcribeLive(requireNotNull(capture), requireNotNull(pipe))
+ }
+
+ override fun sharedObjectDidRelease() {
+ super.sharedObjectDidRelease()
+ cancelInputSelection()
+ capture?.abort()
+ pipe?.close()
+ onRelease(id)
+ }
+}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt
index 8673ce0d790..d8a5c9d7906 100644
--- a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt
@@ -80,6 +80,31 @@ internal fun pacedRecordedResponses(
}
}
+internal fun livePcmResponses(
+ capture: LivePcmCapture,
+ output: PcmOutput,
+ responses: () -> Flow<SpeechRecognizerResponse>
+): Flow<SpeechRecognizerResponse> = flow {
+ try {
+ withTimeout(330000) {
+ responses().collect { emit(it) }
+ requireCompleteAudio(capture.inputExhausted) {
+ "Speech recognition ended before receiving the complete recording."
+ }
+ capture.awaitFinished()
+ output.verifyConsumed()
+ }
+ } catch (e: TimeoutCancellationException) {
+ throw VoiceTranscriptionException(
+ "incomplete-audio",
+ "Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.",
+ e
+ )
+ } finally {
+ capture.cancel()
+ }
+}
+
internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) {
if (!condition) throw VoiceTranscriptionException("incomplete-audio", message())
}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
index b49791bea2b..f193926f1ce 100644
--- a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
@@ -6,7 +6,9 @@ import expo.modules.kotlin.Promise
import expo.modules.kotlin.functions.Coroutine
import expo.modules.kotlin.modules.Module
import expo.modules.kotlin.modules.ModuleDefinition
+import expo.modules.kotlin.modules.ModuleDefinitionBuilder
import java.util.Locale
+import java.util.concurrent.ConcurrentHashMap
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
@@ -22,6 +24,7 @@ class T3VoiceTranscriptionModule : Module() {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val lock = Any()
private var operation: Pair<Int, Job>? = null
+ private val recorders = ConcurrentHashMap<String, LiveVoiceRecorder>()
override fun definition() = ModuleDefinition {
Name("T3VoiceTranscription")
@@ -66,6 +69,27 @@ class T3VoiceTranscriptionModule : Module() {
}
}
+ AsyncFunction("startLive") {
+ id: Int,
+ uri: String,
+ language: String,
+ mode: Int,
+ promise: Promise
+ ->
+ runOperation(id, promise, "transcription-failed") {
+ val recorder = recorders.values.firstOrNull { it.uri == uri }
+ ?: error("The voice recording is no longer available.")
+ val context = requireNotNull(appContext.reactContext)
+ MlKitSpeechClient(context, Locale.forLanguageTag(language), mode).use {
+ recorder.transcribe(uri, it)
+ }
+ }
+ }
+
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+ defineLiveRecorder()
+ }
+
AsyncFunction("cancel") { id: Int, promise: Promise ->
scope.launch {
val job = synchronized(lock) { operation?.takeIf { it.first == id }?.second }
@@ -74,7 +98,37 @@ class T3VoiceTranscriptionModule : Module() {
}
}
- OnDestroy { scope.cancel() }
+ OnDestroy {
+ recorders.values.toList().forEach { it.sharedObjectDidRelease() }
+ scope.cancel()
+ }
+ }
+
+ private fun ModuleDefinitionBuilder.defineLiveRecorder() {
+ Class(LiveVoiceRecorder::class) {
+ Constructor {
+ check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.S)
+ LiveVoiceRecorder(requireNotNull(appContext.reactContext), appContext, scope) {
+ recorders.remove(it)
+ }.also { recorders[it.id] = it }
+ }
+ Property("uri") { recorder -> recorder.uri }
+ AsyncFunction("prepareToRecordAsync") Coroutine { recorder: LiveVoiceRecorder ->
+ withContext(Dispatchers.IO) { recorder.prepare() }
+ }
+ Function("record") { recorder: LiveVoiceRecorder, options: Map<String, Double> ->
+ recorder.record(options["forDuration"] ?: 300.0)
+ }
+ AsyncFunction("stop") Coroutine { recorder: LiveVoiceRecorder -> recorder.stop() }
+ Function("getStatus") { recorder: LiveVoiceRecorder -> recorder.status() }
+ Function("getAvailableInputs") { recorder: LiveVoiceRecorder -> recorder.inputs() }
+ AsyncFunction("setInput") Coroutine { recorder: LiveVoiceRecorder, uid: String ->
+ recorder.setInput(uid)
+ }
+ Function("cancelInputSelection") { recorder: LiveVoiceRecorder ->
+ recorder.cancelInputSelection()
+ }
+ }
}
private fun runOperation(
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LivePcmCaptureTest.kt b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LivePcmCaptureTest.kt
new file mode 100644
index 00000000000..b47c47c37eb
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LivePcmCaptureTest.kt
@@ -0,0 +1,184 @@
+package expo.modules.t3voicetranscription
+
+import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
+import java.io.ByteArrayOutputStream
+import java.io.File
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.CoroutineStart
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class LivePcmCaptureTest {
+ private class Source : PcmCaptureSource {
+ override val capacityBytes = 32000
+ val chunks = Channel<ByteArray>(Channel.UNLIMITED)
+ val reading = Channel<Unit>(Channel.UNLIMITED)
+ var released = false
+ override fun start() = Unit
+ override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
+ reading.send(Unit)
+ val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
+ chunk.copyInto(buffer)
+ return chunk.size
+ }
+ override fun stop() = Unit
+ override fun release() {
+ released = true
+ }
+ }
+
+ private class Output(private val gate: CompletableDeferred<Unit>? = null) : PcmOutput {
+ val bytes = ByteArrayOutputStream()
+ val first = CompletableDeferred<Unit>()
+ val writing = CompletableDeferred<Unit>()
+ val ended = CompletableDeferred<Unit>()
+ var unread = false
+ override suspend fun write(buffer: ByteArray, count: Int) {
+ writing.complete(Unit)
+ gate?.await()
+ bytes.write(buffer, 0, count)
+ first.complete(Unit)
+ }
+ override fun finish() {
+ ended.complete(Unit)
+ }
+ override fun verifyConsumed() = requireCompleteAudio(!unread) { "Unread captured audio." }
+ override fun close() = Unit
+ }
+
+ @Test fun transcribesDuringCaptureAndDrainsTheShortTailAfterStop() = runTest {
+ val source = Source()
+ val output = Output()
+ val file = File.createTempFile("live-capture-test-", ".pcm")
+ val first = ByteArray(3200) { (it % 251).toByte() }
+ val tail = byteArrayOf(1, 2, 3, 4, 5, 6)
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
+ try {
+ capture.start(this, 300.0) {}
+ source.chunks.send(first)
+ output.first.await()
+ assertTrue(capture.isRecording)
+ assertFalse(output.ended.isCompleted)
+ assertArrayEquals(first, output.bytes.toByteArray())
+ source.reading.receive()
+ source.reading.receive()
+
+ val stopped = async(start = CoroutineStart.UNDISPATCHED) { capture.stop() }
+ source.chunks.send(tail)
+ stopped.await()
+ assertTrue(source.released)
+ assertFalse(capture.isRecording)
+ val text = collectGoogleVoiceTranscript(
+ livePcmResponses(capture, output) {
+ flow {
+ output.ended.await()
+ emit(
+ SpeechRecognizerResponse.FinalTextResponse("Opening. Repeat. Repeat.", emptyList())
+ )
+ emit(SpeechRecognizerResponse.FinalTextResponse(" 中間。", emptyList()))
+ emit(SpeechRecognizerResponse.PartialTextResponse(" An unfinished ending"))
+ emit(SpeechRecognizerResponse.PartialTextResponse(" End."))
+ emit(SpeechRecognizerResponse.CompletedResponse)
+ }
+ }
+ )
+ assertEquals("Opening. Repeat. Repeat. 中間。 End.", text)
+ assertArrayEquals(first + tail, output.bytes.toByteArray())
+ assertArrayEquals(first + tail, file.readBytes())
+ } finally {
+ capture.cancel()
+ file.delete()
+ }
+ }
+
+ @Test fun rejectsIncompleteRecognitionAndUnreadCapturedAudio() = runTest {
+ for (failure in listOf("early", "unread", "timeout")) {
+ val early = failure == "early"
+ val source = Source()
+ val output = Output().apply { unread = failure == "unread" }
+ val file = File.createTempFile("live-capture-test-", ".pcm")
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
+ try {
+ capture.start(this, if (early) 300.0 else 0.1) {}
+ source.chunks.send(ByteArray(3200))
+ val result = runCatching {
+ collectGoogleVoiceTranscript(
+ livePcmResponses(capture, output) {
+ flow {
+ if (!early) output.ended.await()
+ if (failure == "timeout") awaitCancellation()
+ emit(SpeechRecognizerResponse.FinalTextResponse("Incomplete.", emptyList()))
+ emit(SpeechRecognizerResponse.CompletedResponse)
+ }
+ }
+ )
+ }
+ assertEquals(
+ "incomplete-audio",
+ (result.exceptionOrNull() as VoiceTranscriptionException).code
+ )
+ assertTrue(source.released)
+ } finally {
+ capture.cancel()
+ file.delete()
+ }
+ }
+ }
+
+ @Test fun durationLimitDrainsQueuedPcmBeforeReleasingTheMicrophone() = runTest {
+ val source = Source()
+ val output = Output()
+ val file = File.createTempFile("live-capture-test-", ".pcm")
+ val expected = ByteArray(6400) { (it % 251).toByte() } + byteArrayOf(42, 0)
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
+ val stopped = CompletableDeferred<Unit>()
+ try {
+ source.chunks.send(expected.copyOfRange(0, 3200))
+ source.chunks.send(expected.copyOfRange(3200, 6400))
+ source.chunks.send(expected.copyOfRange(6400, expected.size))
+ capture.start(this, 0.1) { stopped.complete(Unit) }
+ stopped.await()
+ assertTrue(source.released)
+ assertEquals(expected.size.toLong(), file.length())
+ capture.awaitFinished()
+ assertArrayEquals(expected, output.bytes.toByteArray())
+ } finally {
+ capture.cancel()
+ file.delete()
+ }
+ }
+
+ @Test fun cancellationWaitsForCaptureAndFeederCleanup() = runTest {
+ for (phase in listOf("prepared", "reading", "writing")) {
+ val source = Source()
+ val output = Output(if (phase == "writing") CompletableDeferred() else null)
+ val file = File.createTempFile("live-capture-test-", ".pcm")
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
+ try {
+ if (phase != "prepared") {
+ capture.start(this, 300.0) {}
+ source.reading.receive()
+ if (phase == "writing") {
+ source.chunks.send(ByteArray(3200))
+ output.writing.await()
+ }
+ }
+ capture.cancel()
+ assertTrue(source.released)
+ assertTrue(output.ended.isCompleted)
+ assertTrue(capture.isFinished)
+ assertTrue(runCatching { capture.awaitFinished() }.isFailure)
+ } finally {
+ file.delete()
+ }
+ }
+ }
+}
diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LiveVoiceRecorderTest.kt b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LiveVoiceRecorderTest.kt
new file mode 100644
index 00000000000..6c78053e529
--- /dev/null
+++ b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LiveVoiceRecorderTest.kt
@@ -0,0 +1,123 @@
+package expo.modules.t3voicetranscription
+
+import android.content.Context
+import android.media.AudioRecord
+import android.os.ParcelFileDescriptor
+import com.facebook.react.bridge.BridgeReactContext
+import expo.modules.kotlin.AppContext
+import expo.modules.kotlin.ModulesProvider
+import expo.modules.kotlin.modules.Module
+import java.io.File
+import java.lang.ref.WeakReference
+import kotlinx.coroutines.CoroutineStart
+import kotlinx.coroutines.async
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.RuntimeEnvironment
+import org.robolectric.annotation.Config
+import org.robolectric.util.ReflectionHelpers
+
+@RunWith(RobolectricTestRunner::class)
+@Config(manifest = Config.NONE, sdk = [31])
+class LiveVoiceRecorderTest {
+ private class Source(private val failStop: Boolean = false) : PcmCaptureSource {
+ override val capacityBytes = 32000
+ val chunks = Channel<ByteArray>(Channel.UNLIMITED)
+ val reading = Channel<Unit>(Channel.UNLIMITED)
+ override fun start() = Unit
+ override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
+ reading.send(Unit)
+ val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
+ chunk.copyInto(buffer)
+ return chunk.size
+ }
+ override fun stop() {
+ check(!failStop) { "The microphone could not stop." }
+ }
+ override fun release() = Unit
+ }
+
+ @Test fun preparingAgainReleasesTheUnusedMicrophone() = runTest {
+ withAppContext { context, appContext ->
+ val recorder = LiveVoiceRecorder(context, appContext, this) {}
+ try {
+ recorder.prepare()
+ val previous = ReflectionHelpers.getField<AudioRecord>(recorder, "audio")
+ assertEquals(AudioRecord.STATE_INITIALIZED, previous.state)
+ recorder.prepare()
+ assertEquals(AudioRecord.STATE_UNINITIALIZED, previous.state)
+ } finally {
+ recorder.sharedObjectDidRelease()
+ }
+ }
+ }
+
+ @Test fun stoppingBeforeInferencePreservesQueuedAudioAndClosesUnusedPipes() = runTest {
+ withAppContext { context, appContext ->
+ for (phase in listOf("prepared", "recording", "draining", "prepared-failure")) {
+ val started = phase == "recording" || phase == "draining"
+ val source = Source(phase == "prepared-failure")
+ val pipe = PcmPipe.open()
+ val reader = ReflectionHelpers.getField<ParcelFileDescriptor.AutoCloseInputStream>(
+ pipe,
+ "reader"
+ )
+ val file = File.createTempFile("recorder-stop-test-", ".pcm")
+ val capture = LivePcmCapture(source, file, pipe, testScheduler.timeSource)
+ val recorder = LiveVoiceRecorder(context, appContext, this) {}
+ ReflectionHelpers.setField(recorder, "capture", capture)
+ ReflectionHelpers.setField(recorder, "pipe", pipe)
+ try {
+ assertTrue(reader.fd.valid())
+ if (started) {
+ capture.start(this, 300.0) {}
+ source.reading.receive()
+ source.chunks.send(ByteArray(3200))
+ source.reading.receive()
+ val stopped = async(start = CoroutineStart.UNDISPATCHED) {
+ if (phase == "draining") capture.stop() else recorder.stop()
+ }
+ source.chunks.send(byteArrayOf(1, 2))
+ stopped.await()
+ if (phase == "draining") recorder.stop()
+ assertFalse(capture.isFinished)
+ } else {
+ assertEquals(phase == "prepared-failure", runCatching { recorder.stop() }.isFailure)
+ }
+ assertEquals(started, reader.fd.valid())
+ } finally {
+ capture.cancel()
+ recorder.sharedObjectDidRelease()
+ file.delete()
+ }
+ }
+ }
+ }
+
+ private suspend fun withAppContext(block: suspend (Context, AppContext) -> Unit) {
+ val context = RuntimeEnvironment.getApplication()
+ val reactContext = BridgeReactContext(context)
+ val appContext = AppContext(
+ object : ModulesProvider {
+ override fun getModulesMap() = emptyMap<Class<out Module>, String?>()
+ },
+ expo.modules.core.ModuleRegistry(emptyList(), emptyList()),
+ WeakReference(reactContext)
+ )
+ try {
+ block(context, appContext)
+ } finally {
+ appContext.javaClass.declaredMethods.first {
+ it.name.startsWith("onDestroy") && it.parameterCount == 0
+ }
+ .invoke(appContext)
+ reactContext.destroy()
+ }
+ }
+}
diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
index eab40efd490..ace4f2594a7 100644
--- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
@@ -1,5 +1,5 @@
import { useAtomValue } from "@effect/atom-react";
-import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
+import { RecordingPresets, type RecordingStatus } from "expo-audio";
import { File } from "expo-file-system";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import { useSyncExternalStore } from "react";
@@ -14,6 +14,7 @@ import {
rememberMicrophones,
} from "../../lib/microphonePriority";
import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
+import { createVoiceRecorder } from "../../native/voiceRecorder";
import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
import {
@@ -87,7 +88,7 @@ const voiceInputAtom = Atom.make((get) => {
void session.controller.appMovedToBackground();
});
const recorder = createLazyVoiceRecorder({
- create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
+ create: () => createVoiceRecorder(VOICE_RECORDING_OPTIONS),
onStatus: (status: RecordingStatus) => {
session.controller.handleRecorderStatus({
isFinished: status.isFinished,
diff --git a/apps/mobile/src/native/voiceRecorder.android.ts b/apps/mobile/src/native/voiceRecorder.android.ts
new file mode 100644
index 00000000000..4ac0497d341
--- /dev/null
+++ b/apps/mobile/src/native/voiceRecorder.android.ts
@@ -0,0 +1,8 @@
+import { AudioModule } from "expo-audio";
+import { createLiveVoiceRecorder } from "./voiceTranscription.android";
+
+export function createVoiceRecorder(
+ options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
+) {
+ return createLiveVoiceRecorder() ?? new AudioModule.AudioRecorder(options);
+}
diff --git a/apps/mobile/src/native/voiceRecorder.ts b/apps/mobile/src/native/voiceRecorder.ts
new file mode 100644
index 00000000000..c9f274b3f0b
--- /dev/null
+++ b/apps/mobile/src/native/voiceRecorder.ts
@@ -0,0 +1,7 @@
+import { AudioModule } from "expo-audio";
+
+export function createVoiceRecorder(
+ options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
+) {
+ return new AudioModule.AudioRecorder(options);
+}
diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts
index 7a2e39e4304..e171cf9cb50 100644
--- a/apps/mobile/src/native/voiceTranscription.android.test.ts
+++ b/apps/mobile/src/native/voiceTranscription.android.test.ts
@@ -5,6 +5,17 @@ const mocks = vi.hoisted(() => ({
prepare: vi.fn<(id: number) => Promise<{ locale: string; mode: number }>>(),
transcribe: vi.fn<(id: number, uri: string, locale: string, mode: number) => Promise<string>>(),
cancel: vi.fn<(id: number) => Promise<void>>(),
+ LiveVoiceRecorder: class {
+ readonly uri = "file:///live.pcm";
+ },
+}));
+
+vi.mock("react-native", () => ({
+ Platform: {
+ get Version() {
+ return 31;
+ },
+ },
}));
vi.mock("expo", () => ({
@@ -15,8 +26,17 @@ import {
VoiceInputController,
resetVoiceInputGlobalsForTests,
} from "../../../../packages/client-runtime/src/voice-input/controller";
+import { Platform } from "react-native";
+
+import { createLiveVoiceRecorder, getLocalVoiceTranscriber } from "./voiceTranscription.android";
-import { getLocalVoiceTranscriber } from "./voiceTranscription.android";
+it.each([
+ [30, null],
+ [31, "file:///live.pcm"],
+] as const)("offers live recording only on supported Android versions (%s)", (version, uri) => {
+ using _version = vi.spyOn(Platform, "Version", "get").mockReturnValue(version);
+ expect(createLiveVoiceRecorder()?.uri ?? null).toBe(uri);
+});
it.each([false, true])(
"waits for device support before offering a transcriber (%s)",
diff --git a/apps/mobile/src/native/voiceTranscription.android.ts b/apps/mobile/src/native/voiceTranscription.android.ts
index 3749b6d2a97..78cc33d47b0 100644
--- a/apps/mobile/src/native/voiceTranscription.android.ts
+++ b/apps/mobile/src/native/voiceTranscription.android.ts
@@ -1,4 +1,7 @@
import { requireOptionalNativeModule } from "expo";
+import type { RecorderState, RecordingStatus } from "expo-audio";
+import { Platform } from "react-native";
+import type { NativeVoiceRecorder } from "../features/voice-input/lazyVoiceRecorder";
import {
VoiceTranscriptionError,
@@ -13,6 +16,8 @@ type GoogleTranscriptionModule = {
prepare(id: number): Promise<PreparedGoogleTranscription>;
transcribe(id: number, uri: string, locale: string, mode: number): Promise<string>;
cancel(id: number): Promise<void>;
+ readonly LiveVoiceRecorder?: new () => NativeVoiceRecorder<RecordingStatus, RecorderState>;
+ startLive?(id: number, uri: string, locale: string, mode: number): Promise<string>;
};
const native = requireOptionalNativeModule<GoogleTranscriptionModule>("T3VoiceTranscription");
@@ -33,6 +38,12 @@ const nativeErrorMessages: Readonly<Record<string, string>> = {
let nextOperationId = 0;
+export function createLiveVoiceRecorder() {
+ return Number(Platform.Version) >= 31 && native?.LiveVoiceRecorder
+ ? new native.LiveVoiceRecorder()
+ : null;
+}
+
export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
if (!native || !(await native.checkAvailability())) return null;
const module = native;
@@ -41,8 +52,17 @@ export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | nul
const prepared = await runOperation(module, signal, "preparation-failed", (id) =>
module.prepare(id),
);
+ const startLive = module.startLive?.bind(module);
return {
locale: prepared.locale,
+ ...(module.LiveVoiceRecorder && startLive
+ ? {
+ start: (uri: string, { signal }: { signal: AbortSignal }) =>
+ runOperation(module, signal, "transcription-failed", (id) =>
+ startLive(id, uri, prepared.locale, prepared.mode),
+ ),
+ }
+ : {}),
transcribe: (uri, { signal }) =>
runOperation(module, signal, "transcription-failed", (id) =>
module.transcribe(id, uri, prepared.locale, prepared.mode),
diff --git a/docs/user/composer.md b/docs/user/composer.md
index d1b0562203f..86dff30a79c 100644
--- a/docs/user/composer.md
+++ b/docs/user/composer.md
@@ -198,8 +198,8 @@ voice input isn't available. On Android, keep T3 Code open while recording and
transcribing.
The first use may download a speech model and needs a network connection.
-Later transcription works offline for that language. On Android, transcription
-processes the full recording at roughly its original duration. Recordings can be up to five
+Later transcription works offline for that language. Android processes audio while
+you record, then finishes the transcript after you stop. Recordings can be up to five
minutes long. Canceling or an audio interruption discards the recording and
preserves your existing draft. Navigating to another screen keeps dictation active;
use the floating controls to finish or cancel. While recording, the screen stays
diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts
index 52069d1bd67..a09bfe6f583 100644
--- a/packages/client-runtime/src/voice-input/controller.test.ts
+++ b/packages/client-runtime/src/voice-input/controller.test.ts
@@ -243,6 +243,13 @@ describe("VoiceInputController", () => {
expect(harness.recorder.record).toHaveBeenCalledWith({
forDuration: VOICE_RECORDING_LIMIT_SECONDS,
});
+ await harness.controller.handleRecorderStatus({
+ isFinished: true,
+ hasError: false,
+ error: null,
+ url: "file:///previous-recording.m4a",
+ });
+ expect(harness.controller.currentState.phase).toBe("recording");
const stopping = harness.controller.stop();
harness.controller.handleRecorderStatus({
@@ -342,6 +349,92 @@ describe("VoiceInputController", () => {
expect(events).toEqual(["released", "transcribed"]);
});
+ it("starts streaming transcription during recording and commits only after stopping", async () => {
+ const transcript = deferred<string>();
+ const start = vi.fn(() => transcript.promise);
+ const transcribe = vi.fn(async () => "replayed audio");
+ const harness = createHarness({
+ getTranscriber: () => ({
+ prepare: async () => ({ locale: "en-US", start, transcribe }),
+ }),
+ });
+ await harness.controller.start();
+ expect(start).toHaveBeenCalledWith("file:///voice.m4a", {
+ signal: expect.any(AbortSignal),
+ });
+ expect(harness.controller.currentState.phase).toBe("recording");
+ expect(harness.commits).toEqual([]);
+
+ const stopping = harness.controller.stop();
+ transcript.resolve("opening middle and final words");
+ await stopping;
+ expect(harness.commits[0]?.text).toBe("hello opening middle and final words");
+ expect(transcribe).not.toHaveBeenCalled();
+ expect(harness.deleted).toEqual(["file:///voice.m4a"]);
+ });
+
+ it("waits for canceled live inference before deleting its recording or starting again", async () => {
+ const transcript = deferred<string>();
+ const released = deferred<void>();
+ let signal: AbortSignal | undefined;
+ const harness = createHarness({
+ getTranscriber: () => ({
+ prepare: async () => ({
+ ...preparedTranscription(),
+ start: (_uri: string, options: { signal: AbortSignal }) => {
+ signal = options.signal;
+ return transcript.promise;
+ },
+ }),
+ }),
+ deleteRecording: () => released.resolve(undefined),
+ });
+ await harness.controller.start();
+ expect(signal).toBeDefined();
+ harness.controller.cancel();
+ expect(signal?.aborted).toBe(true);
+ const next = createHarness();
+ const nextStart = next.controller.start();
+ expect(next.recorder.record).not.toHaveBeenCalled();
+
+ transcript.resolve("discard this");
+ await released.promise;
+ await nextStart;
+ expect(harness.commits).toEqual([]);
+ expect(next.controller.currentState.phase).toBe("recording");
+ await next.controller.interruptRecording();
+ });
+
+ it("stops recording after a live inference failure and preserves the draft for retry", async () => {
+ const transcript = deferred<string>();
+ const released = deferred<void>();
+ const harness = createHarness({
+ getTranscriber: () => ({
+ prepare: async () => ({ ...preparedTranscription(), start: () => transcript.promise }),
+ }),
+ deleteRecording: () => released.resolve(undefined),
+ });
+ await harness.controller.start();
+ transcript.reject(
+ new VoiceTranscriptionError({
+ code: "transcription-failed",
+ message: "Speech quota reached.",
+ }),
+ );
+ await released.promise;
+ expect(harness.controller.currentState).toMatchObject({
+ phase: "error",
+ error: "Speech quota reached.",
+ errorAction: "retry",
+ });
+ expect(harness.recorder.stop).toHaveBeenCalled();
+ expect(harness.commits).toEqual([]);
+ const next = createHarness();
+ await next.controller.start();
+ expect(next.controller.currentState.phase).toBe("recording");
+ await next.controller.interruptRecording();
+ });
+
it("ignores a late transcript after the draft owner changes", async () => {
const transcription = deferred<string>();
const transcriptionEntered = deferred<void>();
diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts
index 5b057c61fab..733e37c8a3c 100644
--- a/packages/client-runtime/src/voice-input/controller.ts
+++ b/packages/client-runtime/src/voice-input/controller.ts
@@ -194,6 +194,7 @@ export class VoiceInputController {
private operationToken = 0;
private sessionToken: VoiceInputSession | null = null;
private transcription: PreparedVoiceTranscription | null = null;
+ private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
private transcriptionAbortController: AbortController | null = null;
private capturedDraft: VoiceDraftSnapshot | null = null;
private recordingUri: string | null = null;
@@ -278,6 +279,21 @@ export class VoiceInputController {
return;
}
this.capturedDraft = capturedDraft;
+ if (this.transcription.start && this.recordingUri) {
+ const start = this.transcription.start;
+ const uri = this.recordingUri;
+ this.liveTranscription = runTranscriptionOperation(() =>
+ start(uri, { signal: abortController.signal }),
+ ).then(
+ (text) => ({ text }),
+ (error: unknown) => {
+ if (this.isCurrent(operationToken) && this.state.phase === "recording") {
+ void this.discardRecording(transcriptionErrorMessage(error));
+ }
+ return { error };
+ },
+ );
+ }
this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
this.setState({ phase: "recording", error: null, errorAction: null });
} catch {
@@ -341,6 +357,7 @@ export class VoiceInputController {
handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
if (this.state.phase !== "recording") return;
+ if (status.url && this.recordingUri && status.url !== this.recordingUri) return;
if (status.hasError) {
return this.interruptRecording(
status.error ?? "Voice recording was interrupted.",
@@ -402,9 +419,15 @@ export class VoiceInputController {
const capturedDraft = this.capturedDraft;
let transcript: string;
try {
- transcript = await runTranscriptionOperation(() =>
- transcription.transcribe(recordingUri, { signal }),
- );
+ if (this.liveTranscription) {
+ const result = await this.liveTranscription;
+ if ("error" in result) throw result.error;
+ transcript = result.text;
+ } else {
+ transcript = await runTranscriptionOperation(() =>
+ transcription.transcribe(recordingUri, { signal }),
+ );
+ }
} catch (error) {
if (this.isCurrent(operationToken)) {
this.setError(transcriptionErrorMessage(error), "retry");
@@ -454,6 +477,7 @@ export class VoiceInputController {
}
private async releaseResources(): Promise<void> {
+ if (this.liveTranscription) this.transcriptionAbortController?.abort();
this.rememberRecordingUri(this.recordingUri);
try {
this.rememberRecordingUri(this.dependencies.recorder.uri);
@@ -463,6 +487,8 @@ export class VoiceInputController {
await this.dependencies.recorder.stop();
} catch {}
}
+ await this.liveTranscription;
+ this.liveTranscription = null;
this.recordingUri = null;
for (const uri of this.ownedRecordingUris) {
try {
diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts
index 4a8b1ab4e28..12bfb06e0cd 100644
--- a/packages/client-runtime/src/voice-input/transcription.ts
+++ b/packages/client-runtime/src/voice-input/transcription.ts
@@ -7,6 +7,7 @@ export type VoiceTranscriptionOptions = {
export type PreparedVoiceTranscription = {
readonly locale: string;
+ readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
};
diff --git a/patches/[email protected] b/patches/[email protected]
index 8249a399b3b..c241902a1c7 100644
--- a/patches/[email protected]
+++ b/patches/[email protected]
@@ -339,7 +339,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
return map
}
-+internal fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
++fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
+ val inputs = manager.getDevices(AudioManager.GET_DEVICES_INPUTS).filter {
+ recordingInputType(it.type) != null
+ }
@@ -367,7 +367,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
+ else -> null
+}
+
-+internal fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
++fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
+ val base = recordingInputBaseUid(device)
+ val ambiguous = !isBluetoothInput(device.type) && inputs.any {
+ it.id != device.id && recordingInputBaseUid(it) == base
@@ -401,7 +401,7 @@ index 0000000000000000000000000000000000000000..cdba9724318cf06afd21b4df6b895b38
+import kotlinx.coroutines.withTimeout
+
+@RequiresApi(31)
-+internal class RecordingInputRoute(private val manager: AudioManager) : Closeable {
++class RecordingInputRoute(private val manager: AudioManager) : Closeable {
+ private var previousMode: Int? = null
+
+ suspend fun select(input: AudioDeviceInfo) {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 71ad239cc0d..d1af39291d7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -370,7 +370,7 @@ patchedDependencies:
[email protected]: 07cc8c7e14b3b9af143ddd40a059f117c27e55139fbf20bd5dde2eb30509cedd
[email protected]: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
[email protected]: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
- [email protected]: b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba
+ [email protected]: cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52
[email protected]: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
[email protected]: cd09b6d1744dde4efa081b5619ebcff38877d98f7502a51cab121629dece8237
[email protected]: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -619,7 +619,7 @@ importers:
expo-audio:
specifier: 58.0.9
expo-auth-session:
specifier: ~58.0.13
@@ -18132,7 +18132,7 @@ snapshots:
- supports-color
- typescript
dependencies:
expo: 58.0.14(ecd3689b2bda1172f8915f190f5c7383)