Skip to content

Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.

feat(android): on-device calls in your language, and the soft scroll edge (#155)

leepokaiabd7ba4300af
diff --git a/apps/android/app/src/androidTest/kotlin/com/codync/android/NativeVoiceTest.kt b/apps/android/app/src/androidTest/kotlin/com/codync/android/NativeVoiceTest.kt
index 3b2004a9..bd2f1e52 100644
--- a/apps/android/app/src/androidTest/kotlin/com/codync/android/NativeVoiceTest.kt
+++ b/apps/android/app/src/androidTest/kotlin/com/codync/android/NativeVoiceTest.kt
@@ -61,9 +61,9 @@ class NativeVoiceTest {
val context = InstrumentationRegistry.getInstrumentation().targetContext
assertEquals(PackageManager.PERMISSION_GRANTED, ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO))
ActivityScenario.launch(MainActivity::class.java).use {
- val engine = LocalVoiceAudio(context)
+ val engine = LocalVoiceAudio(context, "")
try { runBlocking { withContext(Dispatchers.Main) {
- val name = engine.initialize(); assertEquals("Offline English · Vosk", name)
+ val name = engine.initialize(); assertTrue(name, name.endsWith(" · Vosk") || name.endsWith(" · Android speech"))
val capturing = CompletableDeferred<Unit>()
engine.listen(1) { event -> if (event is VoiceInput.Level) capturing.complete(Unit) }
withTimeout(10_000) { capturing.await() }
diff --git a/apps/android/app/src/androidTest/kotlin/com/codync/android/PlatformSpeechTest.kt b/apps/android/app/src/androidTest/kotlin/com/codync/android/PlatformSpeechTest.kt
new file mode 100644
index 00000000..dbd3c748
--- /dev/null
+++ b/apps/android/app/src/androidTest/kotlin/com/codync/android/PlatformSpeechTest.kt
@@ -0,0 +1,77 @@
+package com.codync.android
+
+import android.os.Build
+import androidx.test.platform.app.InstrumentationRegistry
+import com.codync.android.core.*
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.Channel
+import org.junit.Assert.*
+import org.junit.Assume.assumeTrue
+import org.junit.Test
+
+/** Android's on-device recognizer hears the pinned speech fixture through the call's own audio path. */
+class PlatformSpeechTest {
+ /** Plays the fixture's PCM in real time, as the microphone would. */
+ private class FixtureMic(private val pcm: ByteArray) : MicSource {
+ private var playing: Job? = null
+ override suspend fun start(frames: (ByteArray, Float) -> Unit, failed: () -> Unit) {
+ playing = CoroutineScope(Dispatchers.IO).launch {
+ // Silence after the speech lets the recognizer finish its last stretch.
+ val audio = pcm + ByteArray(Pcm.RATE * 2 * 2)
+ for (start in audio.indices step 3_200) {
+ frames(audio.copyOfRange(start, minOf(start + 3_200, audio.size)), 0.5f)
+ delay(100)
+ }
+ }
+ }
+ override suspend fun stop() { playing?.cancelAndJoin() }
+ }
+
+ @Test fun androidHearsTheSpeechFixtureOnDevice() = runBlocking {
+ assumeTrue(Build.VERSION.SDK_INT >= 33)
+ val instrumentation = InstrumentationRegistry.getInstrumentation()
+ val context = instrumentation.targetContext
+ val support = PlatformSpeech.support(context)
+ assumeTrue("No installed on-device English: $support", "en-US" in support.installed)
+ val wav = instrumentation.context.assets.open("voice-test.wav").use { it.readBytes() }
+ val input = PlatformSpeechInput(context, "en-US", FixtureMic(samples(wav)))
+ val heard = Channel<String>(Channel.UNLIMITED)
+ val failure = CompletableDeferred<String>()
+ input.listen(1) { event ->
+ when (event) {
+ is VoiceInput.Text -> heard.trySend(event.value)
+ is VoiceInput.Failed -> failure.complete(event.message)
+ is VoiceInput.Level -> { }
+ }
+ }
+ var last = ""
+ try {
+ withTimeout(60_000) {
+ while (!last.contains("one") && !last.contains("1")) {
+ select(heard, failure).let { if (it.second != null) fail(it.second) else last = it.first }
+ }
+ }
+ } finally { input.stop() }
+ assertTrue(last, last.isNotBlank())
+ }
+
+ /** The fixture's 16 kHz mono PCM16 samples: its `data` chunk. */
+ private fun samples(wav: ByteArray): ByteArray {
+ val header = ByteBuffer.wrap(wav).order(ByteOrder.LITTLE_ENDIAN)
+ var offset = 12
+ while (offset + 8 <= wav.size) {
+ val size = header.getInt(offset + 4)
+ if (String(wav, offset, 4, Charsets.US_ASCII) == "data") return wav.copyOfRange(offset + 8, offset + 8 + size)
+ offset += 8 + size + size % 2
+ }
+ error("The fixture has no audio")
+ }
+
+ private suspend fun select(heard: Channel<String>, failure: CompletableDeferred<String>): Pair<String, String?> =
+ kotlinx.coroutines.selects.select {
+ heard.onReceive { it to null }
+ failure.onAwait { "" to it }
+ }
+}
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/AppStoreVoice.kt b/apps/android/app/src/main/kotlin/com/codync/android/AppStoreVoice.kt
index de2de196..5ab406a4 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/AppStoreVoice.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/AppStoreVoice.kt
@@ -68,7 +68,7 @@ private fun AppStore.begin(call: VoiceBinding, provider: VoiceProvider?) {
})
call.recording = { null }
val engine: VoiceEngine = if (provider == null) {
- val audio = LocalVoiceAudio(getApplication())
+ val audio = LocalVoiceAudio(getApplication(), VoicePreferences.language(getApplication()))
call.recording = { audio.recordingSessionId }
VoiceSession(contextScope, audio, settings) { sendFromCall(call, it) }
} else {
@@ -177,6 +177,24 @@ fun AppStore.interruptVoice() { voiceBinding?.engine?.interrupt() }
fun AppStore.resumeVoice() { voiceBinding?.engine?.focus(true) }
internal fun AppStore.focusVoice(active: Boolean) { voiceBinding?.engine?.focus(active) }
internal fun AppStore.voiceRecordingSession(): Int? = voiceBinding?.recording?.invoke()
+/** The language on-device calls hear; empty follows the phone. */
+internal fun AppStore.callLanguage(): String = VoicePreferences.language(getApplication())
+
+/** Picks it, and has Android fetch it when it hears it on device but hasn't downloaded it yet. */
+internal fun AppStore.callLanguage(tag: String) {
+ VoicePreferences.language(getApplication(), tag)
+ if (tag.isNotEmpty()) contextScope.launch {
+ val support = PlatformSpeech.support(getApplication())
+ if (tag !in support.installed && tag in support.downloadable) PlatformSpeech.download(getApplication(), tag)
+ }
+}
+
+/** What on-device calls can hear on this phone: Android's on-device languages, and English always. */
+internal suspend fun AppStore.callLanguages(): List<String> {
+ val onDevice = PlatformSpeech.support(getApplication()).onDevice
+ return if (onDevice.any { VoiceLanguages.base(java.util.Locale.forLanguageTag(it)) == "en" }) onDevice else onDevice + VoiceLanguages.ENGLISH
+}
+
fun AppStore.voiceSettings(settings: VoiceSettings) { VoicePreferences.save(getApplication(), settings); voiceBinding?.engine?.settings(settings) }
// The computer's voice methods: it holds the provider keys and does the provider calls that need them.
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/BotListView.kt b/apps/android/app/src/main/kotlin/com/codync/android/BotListView.kt
index 40d2225f..5924743d 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/BotListView.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/BotListView.kt
@@ -34,8 +34,6 @@ import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.zIndex
import androidx.compose.ui.layout.onSizeChanged
import androidx.compose.ui.draw.clip
-import androidx.compose.ui.graphics.Brush
-import androidx.compose.ui.graphics.Color
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.semantics.CustomAccessibilityAction
import androidx.compose.ui.semantics.Role
@@ -66,9 +64,7 @@ import kotlinx.serialization.json.putJsonArray
val online = state.actualConnection is LinkState.Ready
val top = WindowInsets.statusBars.asPaddingValues().calculateTopPadding()
GlassStage(Modifier.fillMaxSize().background(Palette.background), chrome = {
- // iOS 26 scroll edge: content fades as it slides under the bar.
- Box(Modifier.fillMaxWidth().height(top + 64.dp).background(Brush.verticalGradient(
- 0f to Palette.background, 0.55f to Palette.background.copy(alpha = 0.85f), 1f to Color.Transparent)))
+ ScrollEdge(Edge.Top, top + 64.dp)
Row(Modifier.fillMaxWidth().statusBarsPadding().padding(top = 6.dp).padding(horizontal = 16.dp).height(44.dp),
verticalAlignment = Alignment.CenterVertically) {
AccountSwitcherButton(state, account)
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/ChatScreen.kt b/apps/android/app/src/main/kotlin/com/codync/android/ChatScreen.kt
index 0ca57fe4..5c6c7152 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/ChatScreen.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/ChatScreen.kt
@@ -44,8 +44,6 @@ import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
-import androidx.compose.ui.graphics.Brush
-import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.layout.onSizeChanged
import androidx.compose.ui.platform.LocalContext
@@ -92,8 +90,9 @@ import com.kyant.shapes.Capsule
// The main chat ignores an open thread: it stays put underneath.
val chatState = state.copy(thread = null, trace = false)
GlassStage(Modifier.fillMaxSize().background(Palette.background), chrome = {
- Box(Modifier.fillMaxWidth().height(top + 64.dp).background(Brush.verticalGradient(
- 0f to Palette.background, 0.55f to Palette.background.copy(alpha = 0.85f), 1f to Color.Transparent)))
+ // Down to just above the header's bottom; the bottom one stops under Jump to latest.
+ ScrollEdge(Edge.Top, top + 84.dp)
+ ScrollEdge(Edge.Bottom, with(density) { composerHeight.toDp() } + 4.dp, Modifier.align(Alignment.BottomCenter))
ThreadHeader(bot, state, store, edit, { store.showTrace(true) }, { openRoutine(null) }, { confirmNewSession = true }, { confirmDelete = true })
// The call floats over the chat under the header, as on iOS; the chat stays readable and usable.
AnimatedVisibility(state.voice != null, Modifier.padding(top = top + 54.dp),
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/ConversationList.kt b/apps/android/app/src/main/kotlin/com/codync/android/ConversationList.kt
index df3deeb9..7fc361c6 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/ConversationList.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/ConversationList.kt
@@ -90,8 +90,9 @@ private suspend fun LazyListState.scrollToEnd(animated: Boolean) {
@Composable private fun JumpToLatest(visible: Boolean, modifier: Modifier, action: () -> Unit) {
AnimatedVisibility(visible, modifier, enter = fadeIn(Motion.layout()) + scaleIn(Motion.layout(), initialScale = 0.8f),
exit = fadeOut(Motion.layout()) + scaleOut(Motion.layout(), targetScale = 0.8f)) {
- // iOS `frosted(in: Circle())`: a near-opaque grey disc lifted by a soft shadow.
- val frosted = Color(if (LocalDark.current) 0xFF232323 else 0xFFF4F4F4).copy(alpha = 0.82f)
+ // iOS `frosted(in: Circle())`: grey over a blur, lifted by a soft shadow. Without the blur
+ // underneath (it sits in the content), the grey is nearly opaque so text behind can't show through.
+ val frosted = Color(if (LocalDark.current) 0xFF232323 else 0xFFF4F4F4).copy(alpha = 0.97f)
Box(Modifier.shadow(8.dp, CircleShape).background(frosted, CircleShape)) {
IconButton("Jump to latest", "arrow.down", action = action)
}
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/LocalVoiceAudio.kt b/apps/android/app/src/main/kotlin/com/codync/android/LocalVoiceAudio.kt
index 7530a47e..bf2f80af 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/LocalVoiceAudio.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/LocalVoiceAudio.kt
@@ -4,6 +4,7 @@ import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.media.*
+import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.Looper
@@ -23,21 +24,39 @@ import org.vosk.LibVosk
import org.vosk.Model
import org.vosk.Recognizer
-/** Offline English recognition and an installed local TTS voice. No PCM is written to disk. */
-internal class LocalVoiceAudio(private val context: Context) : VoiceAudio {
+/**
+ * On-device calls: the call's language heard by Android's on-device recognizer, or the bundled English
+ * model, and each reply read by an installed offline voice for the language it's written in. No PCM
+ * is written to disk.
+ */
+internal class LocalVoiceAudio(private val context: Context, private val picked: String) : VoiceAudio {
private val main = Handler(Looper.getMainLooper())
@Volatile private var model: Model? = null
private val mic = AndroidMic(context)
/** Vosk, fed from the microphone's thread; closed only after that thread has stopped. */
private var decoder: Recognizer? = null
+ /** Android's recognizer, when it hears the call's language. */
+ private var platform: PlatformSpeechInput? = null
+ private var language = VoiceLanguages.ENGLISH
private var speech: TextToSpeech? = null
+ private var voices: List<android.speech.tts.Voice> = emptyList()
private var completion: ((Long, String?) -> Unit)? = null
private var speakingId: Long? = null; private var lastChunk: String? = null
val recordingSessionId: Int? get() = mic.recordingSessionId
override suspend fun initialize(): String {
require(ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED)
- withContext(Dispatchers.IO) {
+ val support = PlatformSpeech.support(context)
+ language = VoiceLanguages.resolve(picked, Locale.getDefault(), support.onDevice)
+ val recognition = VoiceLanguages.recognition(language, support.onDevice)
+ ?: throw VoiceUnavailable("Speech recognition isn't available for this language right now.")
+ if (recognition == Recognition.Platform && Build.VERSION.SDK_INT >= 33) {
+ if (language !in support.installed) {
+ PlatformSpeech.download(context, language)
+ throw VoiceUnavailable("Android is getting ${PlatformSpeech.name(language)} speech ready on this phone. Try the call again in a few minutes.")
+ }
+ platform = PlatformSpeechInput(context, language, mic)
+ } else withContext(Dispatchers.IO) {
try {
LibVosk.vosk_set_log_level(-1)
val directory = VoskModels.english(context)
@@ -51,10 +70,9 @@ internal class LocalVoiceAudio(private val context: Context) : VoiceAudio {
val status = withTimeoutOrNull(20_000) { ready.await() }
?: throw VoiceUnavailable("Android's text-to-speech engine didn't start. Try the call again.")
if (status != TextToSpeech.SUCCESS) throw VoiceUnavailable("Install a text-to-speech engine in Android's speech settings.")
- val voices = engine.voices.orEmpty().filter { !it.isNetworkConnectionRequired && it.locale.language == "en" }
- val voice = voices.maxWithOrNull(compareBy<android.speech.tts.Voice> { it.locale == Locale.US }.thenBy { it.quality })
- ?: throw VoiceUnavailable("Install an offline English voice in Android's text-to-speech settings.")
- check(engine.setVoice(voice) == TextToSpeech.SUCCESS)
+ voices = engine.voices.orEmpty().filter { !it.isNetworkConnectionRequired && TextToSpeech.Engine.KEY_FEATURE_NOT_INSTALLED !in it.features.orEmpty() }
+ if (voice(language) == null && voice(VoiceLanguages.ENGLISH) == null)
+ throw VoiceUnavailable("Install an offline voice in Android's text-to-speech settings.")
engine.setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH).build())
engine.setOnUtteranceProgressListener(object : UtteranceProgressListener() {
@@ -72,9 +90,11 @@ internal class LocalVoiceAudio(private val context: Context) : VoiceAudio {
callback?.invoke(id, "Couldn't read the reply. Check the installed offline voice.")
} }
})
- return "Offline English · Vosk"
+ return if (platform != null) "${PlatformSpeech.name(language)} · Android speech" else "Offline English · Vosk"
}
override suspend fun listen(generation: Long, events: (VoiceInput) -> Unit) {
+ val platform = platform
+ if (platform != null && Build.VERSION.SDK_INT >= 33) { platform.listen(generation, events); return }
check(decoder == null)
val recognizer = Recognizer(requireNotNull(model), Pcm.RATE.toFloat()).also { decoder = it }
var previous = ""
@@ -95,17 +115,33 @@ internal class LocalVoiceAudio(private val context: Context) : VoiceAudio {
}
}
override suspend fun stopListening() {
- withContext(NonCancellable) { mic.stop(); decoder?.close(); decoder = null }
+ withContext(NonCancellable) {
+ if (Build.VERSION.SDK_INT >= 33) platform?.stop()
+ mic.stop(); decoder?.close(); decoder = null
+ }
}
override suspend fun speak(id: Long, text: String, rate: Float, completed: (Long, String?) -> Unit) {
val engine = requireNotNull(speech)
val chunks = SpokenText.chunks(text)
speakingId = id; completion = completed; lastChunk = "codync-$id-${chunks.lastIndex}"
+ val written = VoiceLanguages.spoken(text, language)
+ // A reply in a language with no installed voice would come out as noise: say what to install instead.
+ val voice = voice(written) ?: throw VoiceUnavailable("Install an offline ${PlatformSpeech.name(written)} voice in Android's text-to-speech settings to hear this reply.")
+ check(engine.setVoice(voice) == TextToSpeech.SUCCESS)
check(engine.setSpeechRate(rate) == TextToSpeech.SUCCESS)
for ((index, chunk) in chunks.withIndex()) {
check(engine.speak(chunk, TextToSpeech.QUEUE_ADD, Bundle(), "codync-$id-$index") == TextToSpeech.SUCCESS)
}
}
+
+ /** The best installed offline voice for a language: its region (for Chinese, the reader's script), then quality. */
+ private fun voice(tag: String): android.speech.tts.Voice? {
+ val wanted = Locale.forLanguageTag(tag)
+ val base = VoiceLanguages.base(wanted)
+ val region = wanted.country.ifEmpty { if (base == "zh") (if (readsTraditional()) "TW" else "CN") else Locale.getDefault().country }
+ return voices.filter { VoiceLanguages.base(it.locale) == base }
+ .maxWithOrNull(compareBy<android.speech.tts.Voice> { it.locale.country == region }.thenBy { it.quality })
+ }
override suspend fun stopSpeaking() { completion = null; speakingId = null; lastChunk = null; speech?.stop() }
override suspend fun close() {
try { stopListening() }
@@ -148,7 +184,7 @@ internal object VoskModels {
}
}
-/** Call settings on this phone: the pause and reading speed, and the cloud engine choices (`VoiceChoices`). */
+/** Call settings on this phone: the pause, reading speed and language, and the cloud engine choices (`VoiceChoices`). */
internal object VoicePreferences {
private fun stored(context: Context) = context.getSharedPreferences("codync-voice", Context.MODE_PRIVATE)
fun load(context: Context): VoiceSettings {
@@ -158,6 +194,9 @@ internal object VoicePreferences {
fun save(context: Context, settings: VoiceSettings) {
stored(context).edit { putLong("pause", settings.pauseMillis); putFloat("rate", settings.rate) }
}
+ /** The language on-device calls hear (a BCP 47 tag); empty follows the phone. */
+ fun language(context: Context): String = runCatching { stored(context).getString("language", null) }.getOrNull().orEmpty()
+ fun language(context: Context, tag: String) { stored(context).edit { putString("language", tag) } }
fun choices(context: Context): VoiceChoices {
val stored = stored(context)
return VoiceChoices({ runCatching { stored.getString(it, null) }.getOrNull() }) { key, value ->
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/MainTabs.kt b/apps/android/app/src/main/kotlin/com/codync/android/MainTabs.kt
index cc7e4abf..e044c27c 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/MainTabs.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/MainTabs.kt
@@ -51,9 +51,12 @@ private enum class AppTab { Bots, State }
AnimatedVisibility(state.selectedBot == null, Modifier.align(Alignment.BottomCenter),
enter = fadeIn(Motion.fade()) + slideInVertically(Motion.layoutOffset) { it / 2 },
exit = fadeOut(Motion.fade()) + slideOutVertically(Motion.layoutOffset) { it / 2 }) {
- GlassTabBar(tab, listOf(TabItem(AppTab.Bots, "Bots", "bubble.left.and.bubble.right.fill", 33.dp),
- TabItem(AppTab.State, "State", "rectangle.stack.fill")), { tab = it },
- Modifier.padding(bottom = maxOf(bottom - 4.dp, 12.dp)))
+ Box(contentAlignment = Alignment.BottomCenter) {
+ ScrollEdge(Edge.Bottom, barSpace)
+ GlassTabBar(tab, listOf(TabItem(AppTab.Bots, "Bots", "bubble.left.and.bubble.right.fill", 33.dp),
+ TabItem(AppTab.State, "State", "rectangle.stack.fill")), { tab = it },
+ Modifier.padding(bottom = maxOf(bottom - 4.dp, 12.dp)))
+ }
}
}) {
Box(Modifier.fillMaxSize().tabLayer(tab == AppTab.Bots)) {
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/PlatformSpeech.kt b/apps/android/app/src/main/kotlin/com/codync/android/PlatformSpeech.kt
new file mode 100644
index 00000000..98ed7fba
--- /dev/null
+++ b/apps/android/app/src/main/kotlin/com/codync/android/PlatformSpeech.kt
@@ -0,0 +1,145 @@
+package com.codync.android
+
+import android.content.Context
+import android.content.Intent
+import android.icu.util.ULocale
+import android.media.AudioFormat
+import android.os.Build
+import android.os.Bundle
+import android.os.ParcelFileDescriptor
+import android.speech.RecognitionListener
+import android.speech.RecognitionSupport
+import android.speech.RecognitionSupportCallback
+import android.speech.RecognizerIntent
+import android.speech.SpeechRecognizer
+import androidx.annotation.RequiresApi
+import com.codync.android.core.*
+import java.io.IOException
+import java.io.OutputStream
+import java.util.Locale
+import kotlin.coroutines.resume
+import kotlinx.coroutines.*
+
+/** The languages Android's on-device recognizer hears on this phone: ready now, or downloadable. */
+internal data class SpeechSupport(val installed: List<String> = emptyList(), val downloadable: List<String> = emptyList()) {
+ val onDevice: List<String> get() = (installed + downloadable).distinct()
+}
+
+/** Android's on-device speech recognition (Android 13+, where it can be fed the call's own microphone). */
+internal object PlatformSpeech {
+ /** What Android hears on device; nothing before Android 13 or without an on-device recognizer. */
+ suspend fun support(context: Context): SpeechSupport {
+ if (Build.VERSION.SDK_INT < 33 || !SpeechRecognizer.isOnDeviceRecognitionAvailable(context)) return SpeechSupport()
+ return withContext(Dispatchers.Main) {
+ val recognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(context)
+ try {
+ withTimeoutOrNull(5_000) { suspendCancellableCoroutine { done ->
+ recognizer.checkRecognitionSupport(Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), context.mainExecutor,
+ object : RecognitionSupportCallback {
+ override fun onSupportResult(support: RecognitionSupport) {
+ if (done.isActive) done.resume(SpeechSupport(support.installedOnDeviceLanguages,
+ support.supportedOnDeviceLanguages + support.pendingOnDeviceLanguages))
+ }
+ override fun onError(error: Int) { if (done.isActive) done.resume(SpeechSupport()) }
+ })
+ } } ?: SpeechSupport()
+ } finally { recognizer.destroy() }
+ }
+ }
+
+ /** Asks Android to fetch a language it can hear on device but hasn't downloaded yet. */
+ fun download(context: Context, language: String) {
+ if (Build.VERSION.SDK_INT < 33) return
+ val recognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(context)
+ try { recognizer.triggerModelDownload(Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).putExtra(RecognizerIntent.EXTRA_LANGUAGE, language)) }
+ finally { recognizer.destroy() }
+ }
+
+ /** A language's name for the settings, in the phone's language. */
+ fun name(tag: String): String = ULocale.forLanguageTag(tag).getDisplayName(ULocale.getDefault()).ifEmpty { tag }
+}
+
+/**
+ * One utterance at a time through Android's on-device recognizer: the microphone's PCM is written to
+ * it, so the level meter and echo cancelling are the call's own. What it hears so far is reported as
+ * the stretches it has finished plus the one in progress; the call's pause decides when it's sent.
+ */
+@RequiresApi(33)
+internal class PlatformSpeechInput(private val context: Context, private val language: String, private val mic: MicSource) {
+ private var recognizer: SpeechRecognizer? = null
+ private var sink: OutputStream? = null
+ /** The read end, kept open until the call stops: `startListening` hands it over asynchronously. */
+ private var source: ParcelFileDescriptor? = null
+
+ suspend fun listen(generation: Long, events: (VoiceInput) -> Unit) {
+ val (read, write) = ParcelFileDescriptor.createPipe()
+ source = read
+ val output = ParcelFileDescriptor.AutoCloseOutputStream(write).also { sink = it }
+ val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)
+ .putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
+ .putExtra(RecognizerIntent.EXTRA_LANGUAGE, language)
+ .putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
+ .putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true)
+ .putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, read)
+ .putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1)
+ .putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
+ .putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, Pcm.RATE)
+ .putExtra(RecognizerIntent.EXTRA_SEGMENTED_SESSION, RecognizerIntent.EXTRA_AUDIO_SOURCE)
+ withContext(Dispatchers.Main) {
+ recognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(context).apply {
+ setRecognitionListener(Listener(generation, events))
+ startListening(intent)
+ }
+ }
+ try {
+ mic.start({ pcm, level ->
+ events(VoiceInput.Level(generation, level))
+ try { output.write(pcm) } catch (_: IOException) { /* The recognizer let go: stopListening is closing it. */ }
+ }) { events(VoiceInput.Failed(generation, "Couldn't use the microphone. Check its permission and other audio apps.")) }
+ } catch (error: Exception) {
+ stop()
+ events(VoiceInput.Failed(generation, error.message ?: "Couldn't use the microphone."))
+ }
+ }
+
+ suspend fun stop() = withContext(NonCancellable) {
+ mic.stop()
+ val output = sink; sink = null
+ try { withContext(Dispatchers.IO) { output?.close() } } catch (_: IOException) { /* Already closed by the recognizer. */ }
+ withContext(Dispatchers.Main) { recognizer?.cancel(); recognizer?.destroy(); recognizer = null }
+ source?.close(); source = null
+ }
+
+ private inner class Listener(private val generation: Long, private val events: (VoiceInput) -> Unit) : RecognitionListener {
+ /** The stretches finished so far in this utterance. */
+ private var heard = ""
+
+ private fun text(results: Bundle?) = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull().orEmpty()
+ .let { if (VoiceLanguages.base(Locale.forLanguageTag(language)) == "zh") inReadersScript(it) else it }
+
+ override fun onPartialResults(partial: Bundle?) {
+ events(VoiceInput.Text(generation, VoiceLanguages.join(heard, text(partial)), false))
+ }
+ override fun onSegmentResults(segment: Bundle) {
+ heard = VoiceLanguages.join(heard, text(segment))
+ events(VoiceInput.Text(generation, heard, false))
+ }
+ override fun onResults(results: Bundle?) = onSegmentResults(results ?: Bundle())
+ override fun onError(error: Int) {
+ // Silence between stretches isn't a failure: the call's pause sends what was heard.
+ if (error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT) return
+ events(VoiceInput.Failed(generation, when (error) {
+ SpeechRecognizer.ERROR_LANGUAGE_NOT_SUPPORTED, SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE ->
+ "Speech recognition isn't available for this language right now."
+ else -> "Couldn't hear you. Check the microphone and Android's speech settings."
+ }))
+ }
+ override fun onEndOfSegmentedSession() { }
+ override fun onReadyForSpeech(params: Bundle?) { }
+ override fun onBeginningOfSpeech() { }
+ override fun onRmsChanged(rmsdB: Float) { }
+ override fun onBufferReceived(buffer: ByteArray?) { }
+ override fun onEndOfSpeech() { }
+ override fun onEvent(eventType: Int, params: Bundle?) { }
+ }
+}
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/VoiceAudioPorts.kt b/apps/android/app/src/main/kotlin/com/codync/android/VoiceAudioPorts.kt
index 0213bd66..94c72e36 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/VoiceAudioPorts.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/VoiceAudioPorts.kt
@@ -156,10 +156,13 @@ internal object AndroidClipPlayer : ClipPlayer {
* Speech to text writes Chinese in Simplified characters whatever it's told: turn it into Traditional
* for someone who reads Traditional (their first Chinese language, or a Traditional region when none is listed).
*/
-internal fun inReadersScript(text: String): String {
+internal fun inReadersScript(text: String): String =
+ if (readsTraditional()) Transliterator.getInstance("Hans-Hant").transliterate(text) else text
+
+/** Whether the phone's reader reads Chinese in Traditional characters. */
+internal fun readsTraditional(): Boolean {
val languages = LocaleList.getDefault().let { list -> (0 until list.size()).map(list::get) }
val chinese = languages.firstOrNull { it.language == "zh" }
- val traditional = if (chinese != null) ULocale.addLikelySubtags(ULocale.forLocale(chinese)).script == "Hant"
+ return if (chinese != null) ULocale.addLikelySubtags(ULocale.forLocale(chinese)).script == "Hant"
else Locale.getDefault().country in listOf("TW", "HK", "MO")
- return if (traditional) Transliterator.getInstance("Hans-Hant").transliterate(text) else text
}
diff --git a/apps/android/app/src/main/kotlin/com/codync/android/VoiceChatSettings.kt b/apps/android/app/src/main/kotlin/com/codync/android/VoiceChatSettings.kt
index c497f1d9..48e06b44 100644
--- a/apps/android/app/src/main/kotlin/com/codync/android/VoiceChatSettings.kt
+++ b/apps/android/app/src/main/kotlin/com/codync/android/VoiceChatSettings.kt
@@ -47,7 +47,7 @@ import kotlinx.coroutines.launch
}
if (speech) row { PauseRow(settings, ::change) }
} else {
- row { ValueRow("Language", "English", detail = "The language you speak in calls.") }
+ row { LanguageRow(store) }
row {
ValueRow("Reading speed") {
ChoicePicker(settings.rate, listOf(.84f to "Slower", 1f to "Normal", 1.12f to "Faster"),
@@ -62,6 +62,18 @@ import kotlinx.coroutines.launch
}
}
+/** The language on-device calls hear: the phone's, or one Android hears on this phone (English always). */
+@Composable private fun LanguageRow(store: AppStore) {
+ var language by remember { mutableStateOf(store.callLanguage()) }
+ var heard by remember { mutableStateOf(emptyList<String>()) }
+ LaunchedEffect(Unit) { heard = store.callLanguages() }
+ val collator = remember { java.text.Collator.getInstance() }
+ val named = (heard + language).filter { it.isNotEmpty() }.distinct().map { it to PlatformSpeech.name(it) }.sortedWith(compareBy(collator) { it.second })
+ ValueRow("Language", detail = "The language you speak in calls.") {
+ ChoicePicker(language, listOf("" to "System") + named, { language = it; store.callLanguage(it) }, label = "Language")
+ }
+}
+
/** How long a silence ends what you say, before it's sent. */
@Composable private fun PauseRow(settings: VoiceSettings, change: (VoiceSettings) -> Unit) {
ValueRow("Send after a pause of", detail = "Longer leaves time to think mid-sentence.") {
diff --git a/apps/android/core/src/main/kotlin/com/codync/android/core/Voice.kt b/apps/android/core/src/main/kotlin/com/codync/android/core/Voice.kt
index ede85338..47b61b16 100644
--- a/apps/android/core/src/main/kotlin/com/codync/android/core/Voice.kt
+++ b/apps/android/core/src/main/kotlin/com/codync/android/core/Voice.kt
@@ -163,7 +163,12 @@ class VoiceSession(parent: CoroutineScope, private val audio: VoiceAudio, settin
if (next == null) { listen(); return }
utterance++
mutableState.update { it.copy(phase = VoicePhase.Speaking, said = next, heard = "") }
- audio.speak(utterance, next, state.value.settings.rate) { id, error -> post(Event.Done(id, error)) }
+ try { audio.speak(utterance, next, state.value.settings.rate) { id, error -> post(Event.Done(id, error)) } }
+ catch (unavailable: VoiceUnavailable) {
+ // A reply the phone has no voice for stays in chat; the call goes on.
+ mutableState.update { it.copy(error = unavailable.message) }
+ speakNext()
+ }
}
private suspend fun submit() {
val text = state.value.heard.trim()
diff --git a/apps/android/core/src/main/kotlin/com/codync/android/core/VoiceLanguages.kt b/apps/android/core/src/main/kotlin/com/codync/android/core/VoiceLanguages.kt
new file mode 100644
index 00000000..80bdca1d
--- /dev/null
+++ b/apps/android/core/src/main/kotlin/com/codync/android/core/VoiceLanguages.kt
@@ -0,0 +1,69 @@
+package com.codync.android.core
+
+import java.util.Locale
+
+/** How an on-device call hears its language: Android's on-device recognizer, or the bundled English model. */
+enum class Recognition { Platform, Bundled }
+
+/**
+ * The languages of an on-device call (iOS `OnDeviceEngine.languageOptions`), as BCP 47 tags; an empty
+ * pick follows the phone. `onDevice` is what Android's on-device recognizer can hear on this phone.
+ */
+object VoiceLanguages {
+ const val ENGLISH = "en-US"
+
+ /** The call's language: the picked one, else the phone's when Android hears it on device, else English. */
+ fun resolve(picked: String, system: Locale, onDevice: Collection<String>): String =
+ picked.ifEmpty { match(system, onDevice) ?: ENGLISH }
+
+ /** Android's closest on-device language for a locale: the same language and region, then the language. */
+ fun match(locale: Locale, onDevice: Collection<String>): String? {
+ val tags = onDevice.map { it to Locale.forLanguageTag(it) }
+ val language = base(locale)
+ return tags.firstOrNull { (_, it) -> base(it) == language && it.country == locale.country }?.first
+ ?: tags.firstOrNull { (_, it) -> base(it) == language }?.first
+ }
+
+ /** Who hears the language, or null when nothing on this phone can. */
+ fun recognition(tag: String, onDevice: Collection<String>): Recognition? = when {
+ tag in onDevice -> Recognition.Platform
+ base(Locale.forLanguageTag(tag)) == "en" -> Recognition.Bundled
+ else -> null
+ }
+
+ /**
+ * The language to read a reply in, from its writing: Japanese, Korean and Chinese by script; other
+ * writing follows the call's language, or English when the call is in one of those three.
+ */
+ fun spoken(text: String, call: String): String {
+ var han = 0; var kana = 0; var hangul = 0; var other = 0
+ for (char in text) when (Character.UnicodeScript.of(char.code)) {
+ Character.UnicodeScript.HAN -> han++
+ Character.UnicodeScript.HIRAGANA, Character.UnicodeScript.KATAKANA -> kana++
+ Character.UnicodeScript.HANGUL -> hangul++
+ else -> if (char.isLetter()) other++
+ }
+ val language = base(Locale.forLanguageTag(call))
+ return when {
+ kana > 0 && kana + han >= other -> if (language == "ja") call else "ja-JP"
+ hangul > 0 && hangul >= han + other -> if (language == "ko") call else "ko-KR"
+ han > 0 && han >= other -> if (language == "zh") call else "zh"
+ language in listOf("zh", "ja", "ko") -> ENGLISH
+ else -> call
+ }
+ }
+
+ /** Joins recognized stretches: no space between Chinese, Japanese or Korean characters. */
+ fun join(heard: String, next: String): String = when {
+ heard.isEmpty() -> next
+ next.isEmpty() -> heard
+ wide(heard.last()) || wide(next.first()) -> heard + next
+ else -> "$heard $next"
+ }
+
+ /** The language part of a locale, with Mandarin's `cmn` read as `zh`. */
+ fun base(locale: Locale): String = locale.language.let { if (it == "cmn") "zh" else it }
+
+ private fun wide(char: Char) = Character.UnicodeScript.of(char.code) in setOf(Character.UnicodeScript.HAN,
+ Character.UnicodeScript.HIRAGANA, Character.UnicodeScript.KATAKANA, Character.UnicodeScript.HANGUL)
+}
diff --git a/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceLanguagesTest.kt b/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceLanguagesTest.kt
new file mode 100644
index 00000000..f7932bbd
--- /dev/null
+++ b/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceLanguagesTest.kt
@@ -0,0 +1,42 @@
+package com.codync.android.core
+
+import java.util.Locale
+import org.junit.Assert.*
+import org.junit.Test
+
+class VoiceLanguagesTest {
+ private val onDevice = listOf("en-US", "zh-CN", "cmn-Hant-TW", "ja-JP", "fr-FR")
+
+ @Test fun followingThePhoneUsesItsLanguageWhenAndroidHearsItElseEnglish() {
+ assertEquals("cmn-Hant-TW", VoiceLanguages.resolve("", Locale.forLanguageTag("zh-TW"), onDevice))
+ assertEquals("zh-CN", VoiceLanguages.resolve("", Locale.forLanguageTag("zh-SG"), onDevice))
+ assertEquals("fr-FR", VoiceLanguages.resolve("", Locale.forLanguageTag("fr-CA"), onDevice))
+ assertEquals("en-US", VoiceLanguages.resolve("", Locale.forLanguageTag("de-DE"), onDevice))
+ assertEquals("en-US", VoiceLanguages.resolve("", Locale.forLanguageTag("zh-TW"), emptyList()))
+ assertEquals("ja-JP", VoiceLanguages.resolve("ja-JP", Locale.US, onDevice))
+ }
+
+ @Test fun androidHearsItsLanguagesAndTheBundledModelOnlyEnglish() {
+ assertEquals(Recognition.Platform, VoiceLanguages.recognition("ja-JP", onDevice))
+ assertEquals(Recognition.Platform, VoiceLanguages.recognition("en-US", onDevice))
+ assertEquals(Recognition.Bundled, VoiceLanguages.recognition("en-US", emptyList()))
+ assertEquals(Recognition.Bundled, VoiceLanguages.recognition("en-GB", onDevice))
+ assertNull(VoiceLanguages.recognition("de-DE", onDevice))
+ }
+
+ @Test fun repliesAreReadInTheLanguageTheyAreWrittenIn() {
+ assertEquals("cmn-Hant-TW", VoiceLanguages.spoken("好的,我已經修好了。", "cmn-Hant-TW"))
+ assertEquals("zh", VoiceLanguages.spoken("好的,測試都通過了 OK", "en-US"))
+ assertEquals("ja-JP", VoiceLanguages.spoken("テストは全部通りました", "en-US"))
+ assertEquals("ko-KR", VoiceLanguages.spoken("테스트가 통과했습니다", "en-US"))
+ assertEquals("en-US", VoiceLanguages.spoken("All tests pass.", "cmn-Hant-TW"))
+ assertEquals("fr-FR", VoiceLanguages.spoken("Tous les tests passent.", "fr-FR"))
+ }
+
+ @Test fun recognizedStretchesJoinWithSpacesOnlyBetweenWords() {
+ assertEquals("run the tests", VoiceLanguages.join("run the", "tests"))
+ assertEquals("幫我跑測試", VoiceLanguages.join("幫我", "跑測試"))
+ assertEquals("tests", VoiceLanguages.join("", "tests"))
+ assertEquals("run", VoiceLanguages.join("run", ""))
+ }
+}
diff --git a/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceTest.kt b/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceTest.kt
index 81f584fb..ca682789 100644
--- a/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceTest.kt
+++ b/apps/android/core/src/test/kotlin/com/codync/android/core/VoiceTest.kt
@@ -20,10 +20,15 @@ class VoiceTest {
var failedInit = false; var failedClose = false
val spoken = mutableListOf<Pair<Long, String>>()
var done: ((Long, String?) -> Unit)? = null
+ /** Replies containing this have no installed voice. */
+ var unvoiced: String? = null
override suspend fun initialize(): String { if (failedInit) error("Fixture unavailable"); return "Fixture local engine" }
override suspend fun listen(generation: Long, events: (VoiceInput) -> Unit) { this.generation = generation; input = events }
override suspend fun stopListening() { stops++ }
- override suspend fun speak(id: Long, text: String, rate: Float, completed: (Long, String?) -> Unit) { spoken.add(id to text); done = completed }
+ override suspend fun speak(id: Long, text: String, rate: Float, completed: (Long, String?) -> Unit) {
+ if (unvoiced?.let(text::contains) == true) throw VoiceUnavailable("Install an offline Chinese voice")
+ spoken.add(id to text); done = completed
+ }
override suspend fun stopSpeaking() { speakerStops++ }
override suspend fun close() { closed++; if (failedClose) error("Fixture close failure") }
fun text(value: String, final: Boolean = false, owner: Long = generation) { input?.invoke(VoiceInput.Text(owner, value, final)) }
@@ -63,6 +68,18 @@ class VoiceTest {
assertEquals(listOf("After unmute"), sent)
session.end(); runCurrent(); session.awaitEnd()
}
+ @Test fun aReplyWithNoInstalledVoiceIsSkippedWithANoticeAndTheCallGoesOn() = runTest {
+ val audio = Audio().apply { unvoiced = "測試" }
+ val session = VoiceSession(backgroundScope, audio, VoiceSettings()) { }
+ runCurrent()
+ session.speak("測試通過了"); session.speak("Second reply"); runCurrent()
+ assertEquals("Install an offline Chinese voice", session.state.value.error)
+ assertEquals(listOf("Second reply"), audio.spoken.map { it.second })
+ assertEquals(VoicePhase.Speaking, session.state.value.phase)
+ audio.done?.invoke(audio.spoken.single().first, null); runCurrent()
+ assertEquals(VoicePhase.Listening, session.state.value.phase)
+ session.end(); runCurrent(); session.awaitEnd()
+ }
@Test fun repliesPauseTheMicrophoneAndStalePlaybackCompletionCannotRestartIt() = runTest {
val audio = Audio(); val session = VoiceSession(backgroundScope, audio, VoiceSettings()) { fail("Playback audio must not submit text") }
runCurrent(); val recognition = audio.generation
diff --git a/apps/android/design/src/main/kotlin/com/codync/android/design/Controls.kt b/apps/android/design/src/main/kotlin/com/codync/android/design/Controls.kt
index 74a93cdb..dc5edbcc 100644
--- a/apps/android/design/src/main/kotlin/com/codync/android/design/Controls.kt
+++ b/apps/android/design/src/main/kotlin/com/codync/android/design/Controls.kt
@@ -18,6 +18,8 @@ import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
+import androidx.compose.ui.layout.Layout
+import androidx.compose.ui.unit.Constraints
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxWithConstraints
@@ -199,7 +201,7 @@ import com.kyant.shapes.RoundedRectangle
enabled = enabled, label = label?.let { "$it: $current" }) {
Row(Modifier.then(if (fill == Color.Transparent) Modifier.outline() else Modifier).pill(fill),
verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(6.dp)) {
- Text(current, style = Type.body, color = Palette.text, maxLines = 1, overflow = TextOverflow.Ellipsis)
+ Text(current, Modifier.weight(1f, fill = false), style = Type.body, color = Palette.text, maxLines = 1, overflow = TextOverflow.Ellipsis)
Symbol("chevron.down", size = 13.dp, tint = Palette.secondary)
}
}
@@ -253,14 +255,26 @@ class CardRows internal constructor() {
Box(modifier.fillMaxWidth().height(px).background(Palette.border))
}
-/** Label on the left, value on the right: the replacement for `LabeledContent`. */
+/**
+ * Label on the left, value on the right: the replacement for `LabeledContent`. A long value (a
+ * language's full name) takes at most 60% of the row and truncates, so the label never breaks mid-word.
+ */
@Composable fun ValueRow(label: String, modifier: Modifier = Modifier, detail: String? = null, value: @Composable () -> Unit) {
- Row(modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp)) {
- Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(3.dp)) {
+ Layout({
+ Column(verticalArrangement = Arrangement.spacedBy(3.dp)) {
Text(label, style = Type.body, color = Palette.text)
if (detail != null) Text(detail, style = Type.caption.copy(lineHeight = Type.caption.lineHeight * 1.12f), color = Palette.secondary)
}
CompositionLocalProvider(LocalContentColor provides Palette.secondary, LocalTextStyle provides Type.body.copy(color = Palette.secondary)) { value() }
+ }, modifier.fillMaxWidth()) { (labels, values), constraints ->
+ val width = constraints.maxWidth
+ val shown = values.measure(Constraints(maxWidth = (width * 0.6f).toInt()))
+ val text = labels.measure(Constraints(maxWidth = (width - shown.width - 12.dp.roundToPx()).coerceAtLeast(0)))
+ val height = maxOf(text.height, shown.height)
+ layout(width, height) {
+ text.place(0, (height - text.height) / 2)
+ shown.place(width - shown.width, (height - shown.height) / 2)
+ }
}
}
diff --git a/apps/android/design/src/main/kotlin/com/codync/android/design/Glass.kt b/apps/android/design/src/main/kotlin/com/codync/android/design/Glass.kt
index bccbd71b..10a68144 100644
--- a/apps/android/design/src/main/kotlin/com/codync/android/design/Glass.kt
+++ b/apps/android/design/src/main/kotlin/com/codync/android/design/Glass.kt
@@ -5,6 +5,12 @@ import androidx.compose.foundation.interaction.collectIsPressedAsState
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxScope
import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.ui.graphics.CompositingStrategy
+import androidx.compose.ui.graphics.Brush
+import androidx.compose.ui.graphics.BlendMode
+import androidx.compose.ui.draw.drawWithContent
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.animation.core.animateFloatAsState
import androidx.compose.runtime.Composable
import androidx.compose.runtime.CompositionLocalProvider
@@ -53,6 +59,32 @@ val LocalBackdrop = staticCompositionLocalOf<Backdrop> { emptyBackdrop() }
}
}
+/** Which bar a scroll edge sits under. */
+enum class Edge { Top, Bottom }
+
+/**
+ * iOS 26's soft scroll edge under a bar: content passing beneath blurs and washes into the
+ * background, strongest at the screen's edge, so the bar stays legible over it. Chrome of a
+ * `GlassStage`, like the bar it sits under.
+ */
+@Composable fun ScrollEdge(edge: Edge, height: Dp, modifier: Modifier = Modifier) {
+ val backdrop = LocalBackdrop.current
+ val ground = Palette.background
+ val top = edge == Edge.Top
+ Box(modifier.fillMaxWidth().height(height)
+ .graphicsLayer { compositingStrategy = CompositingStrategy.Offscreen }
+ .drawWithContent {
+ drawContent()
+ // The blur and the wash both fade out toward the content.
+ drawRect(Brush.verticalGradient(*if (top) arrayOf(0f to Color.Black, 0.8f to Color.Black, 1f to Color.Transparent)
+ else arrayOf(0f to Color.Transparent, 0.2f to Color.Black, 1f to Color.Black)), blendMode = BlendMode.DstIn)
+ }
+ .drawPlainBackdrop(backdrop, shape = { RoundedRectangle(0.dp) }, effects = { blur(10.dp.toPx()) }, onDrawSurface = {
+ val wash = listOf(ground.copy(alpha = 0.85f), ground.copy(alpha = 0.45f))
+ drawRect(Brush.verticalGradient(if (top) wash else wash.reversed()))
+ }))
+}
+
/** How strongly glass frosts what's behind it. */
enum class GlassStyle { Regular, Clear, Menu }
diff --git a/docs/design/ui-conventions.md b/docs/design/ui-conventions.md
index c76f5ef7..adc5d033 100644
--- a/docs/design/ui-conventions.md
+++ b/docs/design/ui-conventions.md
@@ -53,7 +53,7 @@ The iPhone and the desktop no longer share UI code. `CodyncUI` (`apps/ios/Kit/`)
- The iPhone chat's chrome is in `ThreadView.swift` (`body`, `chrome`, `header`). The iPhone conversation (chat and thread) is a UIKit collection view of hosted SwiftUI rows (`ConversationList.swift`): it loads earlier messages from the computer as the reader scrolls up, runs under the bars and the composer (the list sits in the safe area and reaches past it, so rows never get a safe area of their own), follows the newest message until they scroll away (with a round "Jump to latest" button), and pops each new message in ([chat messages](../features/chat-messages.md)).
- The desktop UI rules (custom controls, no system menus except the tray, icon-only buttons, fill or border) are implemented in `apps/desktop/src/renderer/components/` (`Controls.tsx`, `Overlay.tsx`, `Icon.tsx`).
-- The Android app (`apps/android/`) ports the iPhone screens one to one in Compose. Where iOS uses system UI (the tab bar, menus, sheets, Liquid Glass), Android draws the same look with its `design` module: glass from Kyant's backdrop library (blur, lens refraction and highlight on RenderEffect/RuntimeShader), the iOS sheet with its grabber and drag-to-close, glass menus and context menus, Inter for SF Pro and Material Symbols for SF Symbols (`Symbol("sf.name")`). Android has no Liquid Glass of its own; Material widgets aren't used.
+- The Android app (`apps/android/`) ports the iPhone screens one to one in Compose. Where iOS uses system UI (the tab bar, menus, sheets, Liquid Glass), Android draws the same look with its `design` module: glass from Kyant's backdrop library (blur, lens refraction and highlight on RenderEffect/RuntimeShader), the iOS sheet with its grabber and drag-to-close, glass menus and context menus, Inter for SF Pro and Material Symbols for SF Symbols (`Symbol("sf.name")`), and the soft scroll edge under its bars (`ScrollEdge`: the content behind the header, composer and tab bar blurs and washes out). Android has no Liquid Glass of its own; Material widgets aren't used.
- A change to one lands in the other and the TUI in the same change; see [native client parity](#native-client-parity).
## Shared application controls
diff --git a/docs/features/voice-call.md b/docs/features/voice-call.md
index 4c577556..d7a638fa 100644
--- a/docs/features/voice-call.md
+++ b/docs/features/voice-call.md
@@ -8,7 +8,7 @@ The bot is the same agent on the computer; the call only changes how you talk to
## On-device voice
- `apps/ios/Kit/Sources/CodyncUI/Call/` (iPhone) and `apps/desktop/src/renderer/views/call/` (desktop): `CallView` (the call bar) over a `VoiceEngine`; `OnDeviceEngine` is the default audio loop. On the Mac the desktop app runs speech recognition in `codync-speech` (`apps/desktop/native/speech-macos/`, driven by `apps/desktop/src/main/speech.ts`).
-- Android: `CallBar` (`apps/android/app/.../VoiceControls.kt`) over the same `VoiceEngine` set, ported into `apps/android/core` (`VoiceEngine.kt`, `CloudSpeechEngine.kt`, `GeminiLiveEngine.kt`, `OpenAIRealtimeEngine.kt`, `CallOperator.kt`) with the phone's audio in the app (`VoiceAudioPorts.kt`, `WebRtcPeer.kt`). On device is `VoiceSession` with offline English Vosk recognition and an installed Android voice (`LocalVoiceAudio.kt`); `VoiceService` (a microphone foreground service) holds audio focus and the Bluetooth/wired route for the call. Settings keys match the iPhone's (`callEngine`, `callMode.<provider>`, …).
+- Android: `CallBar` (`apps/android/app/.../VoiceControls.kt`) over the same `VoiceEngine` set, ported into `apps/android/core` (`VoiceEngine.kt`, `CloudSpeechEngine.kt`, `GeminiLiveEngine.kt`, `OpenAIRealtimeEngine.kt`, `CallOperator.kt`) with the phone's audio in the app (`VoiceAudioPorts.kt`, `WebRtcPeer.kt`). On device is `VoiceSession` (`LocalVoiceAudio.kt`): the call's language (Voice chat settings → Language, the phone's by default) is heard by Android's on-device recognizer on Android 13+, fed the call's own microphone (`PlatformSpeech.kt`; Android asks before downloading a language), with the bundled Vosk model for English where Android has none; each reply is read by an installed offline voice for the language it's written in, and a reply with no installed voice stays in chat with a notice; `VoiceService` (a microphone foreground service) holds audio focus and the Bluetooth/wired route for the call. Settings keys match the iPhone's (`callEngine`, `callMode.<provider>`, …).
- UI follows Grok Bot: a floating capsule over the top of the chat (`ThreadView` overlay), and the
chat stays readable and usable underneath. Left to right: the bot's avatar (pulses while
speaking; tap it to interrupt), a dotted level line (your voice while listening, a ripple while
diff --git a/docs/plans/android-app.md b/docs/plans/android-app.md
index be7a1f26..a129c59b 100644
--- a/docs/plans/android-app.md
+++ b/docs/plans/android-app.md
@@ -22,7 +22,7 @@ emulator against a real host; device acceptance is listed separately.
| Memory | Ported: search, filters, add/edit, pin, review, history and timeline, export/import, forget all | — |
| Routines | Ported: list with switches, editor (schedule, webhook panel, run settings), test run, delete | Live provider/routine acceptance |
| Extensions | Ported: agents (install, sign-in, setup terminal), registry connectors, Composio apps, skills, installed list, custom connector, own skill, credentials, chat connection request card | Real provider OAuth, 1Password |
-| Voice | Ported: call bar (provider name, fallback notice), Voice chat settings (engine, mode, models, voice, minutes this month), on device (Vosk English + Android TTS), OpenAI Realtime (WebRTC), Gemini Live, Speech mode, fallback to on device, group calls | Both providers with a real key on a device; on-device recognition in languages other than English |
+| Voice | Ported: call bar (provider name, fallback notice), Voice chat settings (engine, mode, models, voice, minutes this month), on device (the call's language through Android's on-device recognizer, Vosk English as the fallback, replies read by an offline voice for their language), OpenAI Realtime (WebRTC), Gemini Live, Speech mode, fallback to on device, group calls | Both providers with a real key on a device; a live microphone on a device |
| State/widgets | Ported: Widget (with the animated Add a widget guide), Live update and Status bar pages, Usage sheet, Choose the look; Provider usage, Usage limits, Bots and Team widgets with launcher Edit widget and picker previews (Android 15+); the Bots Quick Settings tile (iOS's Control Center control) | Physical launcher refresh; the tile opens Codync, not Computers, when no bot is at work |
| Notifications | Ported: alerts as conversations with the bot's face (a group's face and its speaking member), conversation shortcuts, what happened ("Task complete") as sub text | Android's conversation layout doesn't draw sub text; background delivery needs FCM in the deployment |
| Live updates | Android 16 Live updates (`ProgressStyle`), the status bar chip on Android 16 QPR1 (API 36.1) | Background updates need FCM in the deployment |