Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(voice): add Google ASR and TTS

kuafu-limanxiangda96d8f60a20
diff --git a/app/build.gradle b/app/build.gradle
index bf71cf2..e1a3d2a 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -51,6 +51,26 @@ android {
buildConfigField "String", "DEFAULT_LANG", buildConfigString('DEFAULT_LANG')
buildConfigField "String", "DEFAULT_THINKING", buildConfigString('DEFAULT_THINKING')
+ // The Google voice path, baked in at package time.
+ //
+ // Same rule as CODEFLYING_KEY above and for the same reason: a key in
+ // the source is a key in a search index, so it lives in
+ // local.properties — which .gitignore excludes — and the repository's
+ // copy of that file stays empty. The source's own default is the empty
+ // string, so a checkout with nothing configured builds exactly what it
+ // built before Google existed.
+ //
+ // **GOOGLE_VOICE_KEY non-empty is also what makes 谷歌 the default
+ // provider** — see VoiceConfig.voiceProvider. A build is packaged for
+ // whoever it is packaged for, and baking a key is the most explicit
+ // statement of that intent there is.
+ buildConfigField "String", "GOOGLE_VOICE_KEY", buildConfigString('GOOGLE_VOICE_KEY')
+ buildConfigField "String", "GOOGLE_VOICE_PROJECT", buildConfigString('GOOGLE_VOICE_PROJECT')
+ buildConfigField "String", "GOOGLE_VOICE_LOCATION", buildConfigString('GOOGLE_VOICE_LOCATION')
+ buildConfigField "String", "GOOGLE_VOICE_BASE_URL", buildConfigString('GOOGLE_VOICE_BASE_URL')
+ buildConfigField "String", "GOOGLE_VOICE_ASR_MODEL", buildConfigString('GOOGLE_VOICE_ASR_MODEL')
+ buildConfigField "String", "GOOGLE_VOICE_TTS_MODEL", buildConfigString('GOOGLE_VOICE_TTS_MODEL')
+
// Verification hook, off unless asked for: `-PdbgSuffix=.dbg` builds a
// *parallel* package that installs next to whatever is already on the
// device. Needed because the package on the tablet was signed with a
diff --git a/app/src/debug/res/xml/network_security_config.xml b/app/src/debug/res/xml/network_security_config.xml
new file mode 100644
index 0000000..3cf4beb
--- /dev/null
+++ b/app/src/debug/res/xml/network_security_config.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+ **仅 debug。** 资源合并时同名资源由构建变体胜出,所以这份整个盖掉
+ src/main/res/xml/network_security_config.xml —— release 包里跑的仍然
+ 是那份严格的(只放行一个主机),发布包不会因为联调多出任何明文口子。
+
+ 这里多出来的是回环地址,给本机联调用的:
+
+ adb reverse tcp:8080 tcp:8080
+
+ 把平板的 127.0.0.1:8080 经 USB 转回开发机,两边不在同一个网也能连。
+ 从 app 的角度看这就是自己机器上的地址,但 Android 的明文策略不区分这个,
+ 不写就报 "CLEARTEXT communication to 127.0.0.1 not permitted by
+ network security policy" —— 而且这句话会一路冒到界面上,看着像版本检查
+ 失败,跟网络配置八竿子打不着。
+
+ **改 src/main 那份的时候记得同步这里。** 是整文件覆盖,不是追加 ——
+ 漏一个主机,debug 上就会少一个能连的地址,而 release 上一切正常。
+-->
+<network-security-config>
+ <domain-config cleartextTrafficPermitted="true">
+ <domain includeSubdomains="true">124.71.176.202</domain>
+ </domain-config>
+
+ <domain-config cleartextTrafficPermitted="true">
+ <domain includeSubdomains="true">127.0.0.1</domain>
+ <domain includeSubdomains="true">localhost</domain>
+ </domain-config>
+</network-security-config>
diff --git a/app/src/main/java/net/kuafuai/andee/asr/AsrController.kt b/app/src/main/java/net/kuafuai/andee/asr/AsrController.kt
index 337d1a7..b7e70be 100644
--- a/app/src/main/java/net/kuafuai/andee/asr/AsrController.kt
+++ b/app/src/main/java/net/kuafuai/andee/asr/AsrController.kt
@@ -26,8 +26,8 @@ import java.util.concurrent.Executors
* ├── error → subtitle ERROR, IDLE
* └── toggle() → IDLE (user cancelled before final)
*
- * All 火山 WS callbacks arrive on the client's IO thread; we marshal them to
- * the UI thread via FloatingWindowUi's own Handler.
+ * Every engine's callbacks arrive on its own thread, never the main one; we
+ * marshal them to the UI thread via FloatingWindowUi's own Handler.
*/
class AsrController(
private val context: Context,
@@ -49,7 +49,7 @@ class AsrController(
},
) {
@Volatile
- private var asr: HuoshanAsr? = null
+ private var asr: AsrEngine? = null
private val executor = Executors.newSingleThreadExecutor { r ->
Thread(r, "AsrController").apply { isDaemon = true }
}
@@ -75,18 +75,45 @@ class AsrController(
private val silence = Handler(Looper.getMainLooper())
private val silenceWatchdog = Runnable { checkSilence() }
- /** When the recognizer last reported *new* words. Written from the 火山 IO thread. */
+ /** When the recognizer last reported *new* words. Written from the engine's own thread. */
@Volatile
private var lastSpeechAt = 0L
/**
- * Last partial text we counted. Written from the 火山 IO thread. Guards
+ * Last partial text we counted. Written from the engine's own thread. Guards
* against a recognizer that re-sends the same hypothesis while the user is
* silent — without it the watchdog would never fire.
*/
@Volatile
private var lastPartial = ""
+ /** When this turn began, for the [MAX_TURN_MS] ceiling. */
+ private var turnStartedAt = 0L
+
+ /**
+ * Set by [AsrEngine.Listener.onReady]; false until the engine can hear.
+ * See [checkSilence] for why the clock waits on it.
+ */
+ @Volatile
+ private var engineReady = false
+
+ /**
+ * Has the recogniser reported anything at all this turn. A plain flag
+ * rather than a timestamp comparison, which is what it used to be: the two
+ * timestamps involved are set by different code paths now, so comparing
+ * them for equality would have been true only by luck.
+ */
+ @Volatile
+ private var heardAnything = false
+
+ /**
+ * How long a silence ends this turn, as answered by the engine in play —
+ * see [AsrEngine.silenceMs]. Seeded with 火山's value so that the watchdog
+ * has something sane to use in the window before an engine is built.
+ */
+ @Volatile
+ private var turnSilenceMs = 3000L
+
fun isActive(): Boolean = audio.isStreaming()
/** Called from the UI thread when the user taps the ball. */
@@ -103,8 +130,14 @@ class AsrController(
}
}
- fun stop() {
- executor.execute { runCatching { stopImpl() } }
+ /**
+ * [bySilence] is true only from [checkSilence] — the caller's own silence
+ * rule is the reason, which an engine uses to decide how long its last
+ * results can possibly still be worth waiting for. See
+ * [AsrEngine.endOfTurn].
+ */
+ fun stop(bySilence: Boolean = false) {
+ executor.execute { runCatching { stopImpl(bySilence) } }
}
fun shutdown() {
@@ -114,15 +147,32 @@ class AsrController(
private fun startImpl() {
val cfg = VoiceConfig.load(context)
+ // Set here, not in [armSilenceWatchdog], because 火山 reports
+ // [AsrEngine.Listener.onReady] from *inside* its own connect — before
+ // the capture starts and before that method runs. Measuring the
+ // connection from a timestamp this turn had not set yet produced
+ // "engine ready after 206230301ms": the process's uptime, reported as
+ // if it were a handshake.
+ turnStartedAt = SystemClock.uptimeMillis()
+ heardAnything = false
// First thing, so the media app has the whole 火山 handshake to actually
// stop in — asking and then opening the mic in the same breath would
// still catch the tail of whatever was playing. A phone call takes the
// floor back: end the turn rather than transcribe one side of a call.
focus.acquire { stop() }
- val listener = object : HuoshanAsr.Listener {
+ val listener = object : AsrEngine.Listener {
+ override fun onReady() {
+ // The session can hear. Start the silence clock here, not at the
+ // microphone — see [checkSilence].
+ Log.i(TAG, "engine ready after ${SystemClock.uptimeMillis() - turnStartedAt}ms")
+ lastSpeechAt = SystemClock.uptimeMillis()
+ engineReady = true
+ }
+
override fun onPartial(text: String) {
if (text.isEmpty()) return
+ heardAnything = true
window.setSubtitle(text, FloatingWindowUi.SubtitleKind.PARTIAL)
if (text != lastPartial) {
lastPartial = text
@@ -160,8 +210,35 @@ class AsrController(
override fun onClosed() {}
}
- val client = HuoshanAsr(cfg, listener)
- client.start(sampleRate = 16_000, channels = 1)
+ // Which service hears. The pick is a string and the failure direction
+ // is 火山 — see [VoiceConfig.voiceProvider] — so an unrecognised value
+ // can only ever mean "keep doing what this device already did".
+ val client: AsrEngine = when (cfg.voiceProvider) {
+ VoiceConfig.VOICE_GOOGLE -> GoogleAsr(cfg, listener)
+ else -> HuoshanAsr(cfg, listener)
+ }
+ turnSilenceMs = client.silenceMs
+ Log.i(
+ TAG,
+ "engine: silence=${client.silenceMs}ms finalWait=${client.finalWaitMs}ms",
+ )
+ // **Cleared here, before the engine exists, and not in [armSilenceWatchdog].**
+ // 火山 reports [AsrEngine.Listener.onReady] from inside its own connect —
+ // its `start` blocks on the handshake and calls it before returning — so
+ // clearing this anywhere after that would erase the readiness the engine
+ // had just declared, and the silence rule would never fire: the
+ // microphone would sit open until [MAX_TURN_MS].
+ engineReady = false
+
+ // **The microphone and the tap's acknowledgement come first; the
+ // handshake follows them.** 火山's `start` blocks on its own connect —
+ // 1.7–2.2 s measured, and it is the CodeFlying path as much as the
+ // direct one, since the gateway speaks the same protocol — so under the
+ // old order the ball sat unchanged for two seconds after the tap and
+ // the microphone did not open until it was over. The first words of the
+ // sentence were never captured at all. Google's engine never had this,
+ // because its `start` returns at once and its `sendAudio` holds what
+ // arrives early; [HuoshanAsr] now does the same, so the two agree.
asr = client
window.setState(FloatingWindowUi.State.RECORDING)
window.setSubtitle(
@@ -177,6 +254,14 @@ class AsrController(
runCatching { asr?.sendAudio(chunk, isLast = false) }
.onFailure { Log.w(TAG, "sendAudio failed", it) }
}
+
+ client.start(sampleRate = 16_000, channels = 1)
+
+ // **Armed after the capture starts, and that ordering is load-bearing
+ // too.** [checkSilence] opens by giving up when there is no stream — the
+ // normal case being a turn that has already ended — so arming this
+ // before [audio.startStreaming] means the very first tick, 250 ms in,
+ // finds no stream and stops the watchdog permanently.
armSilenceWatchdog()
}
@@ -196,48 +281,116 @@ class AsrController(
}
/**
- * Main thread only. Nothing new out of the recognizer for [SILENCE_MS] and
- * we close the mic ourselves — [stop] sends `isLast`, waits for 火山's final
- * line, and that final line is what reaches the brain and starts the task.
- * Indistinguishable from the user tapping again, which is the point.
+ * Main thread only. Nothing new out of the recogniser for the engine's own
+ * [AsrEngine.silenceMs] and we close the mic ourselves — [stop] says the
+ * turn is over, waits for the engine's final line, and that final line is
+ * what reaches the brain and starts the task. Indistinguishable from the
+ * user tapping again, which is the point.
+ *
+ * **The clock is fed by the recogniser's own output and by nothing else,
+ * and that is a simplification made after three attempts to do better.**
+ * A local energy gate was added, tuned, re-tuned, extended to the gateway
+ * path, and finally deleted: every version of it was a threshold that
+ * separated the user from their room, and every room is different. The
+ * device has a better answer available — the recogniser is already telling
+ * us what it hears, at 火山's few-hundred-millisecond cadence and at
+ * roughly one result a second from Gemini — so the only thing the window
+ * has to exceed is that cadence, not any property of the room.
+ *
+ * Two rules hold the edges, and neither needs calibrating:
+ *
+ * * **Nothing counts until the engine can hear** — [AsrEngine.Listener.onReady].
+ * The microphone opens 1.4–1.9 s before the session comes up on this
+ * device, and a window measured from the former used to expire on top of
+ * the handshake.
+ * * **A turn that has heard nothing at all gets [SILENT_START_GRACE_MS]
+ * before the silence rule applies.** Otherwise a tap followed by silence
+ * would end [MAX_TURN_MS] later, and a recogniser whose first result is
+ * slow would be cut off before it ever spoke.
*/
private fun checkSilence() {
if (!audio.isStreaming()) return // finished already
- val quiet = SystemClock.uptimeMillis() - lastSpeechAt
- if (quiet < SILENCE_MS) {
+ val now = SystemClock.uptimeMillis()
+ if (now - turnStartedAt >= MAX_TURN_MS) {
+ Log.i(TAG, "turn hit ${MAX_TURN_MS}ms ceiling — auto-stop")
+ runCatching { stop() }
+ return
+ }
+ if (!engineReady) {
silence.postDelayed(silenceWatchdog, SILENCE_CHECK_MS)
return
}
- Log.i(TAG, "silence ${quiet}ms — auto-stop")
- runCatching { stop() }
+
+ // Nothing heard yet: no partial has ever arrived, so there is nothing
+ // to have been silent *after*. Bounded, or a tap on the ball followed
+ // by nothing would hold the microphone to [MAX_TURN_MS].
+ if (!heardAnything && now - turnStartedAt < SILENT_START_GRACE_MS) {
+ silence.postDelayed(silenceWatchdog, SILENCE_CHECK_MS)
+ return
+ }
+
+ val quiet = now - lastSpeechAt
+ if (quiet < turnSilenceMs) {
+ silence.postDelayed(silenceWatchdog, SILENCE_CHECK_MS)
+ return
+ }
+ Log.i(TAG, "silence ${quiet}ms (limit=$turnSilenceMs) — auto-stop")
+ runCatching { stop(bySilence = true) }
}
- private fun stopImpl() {
+ private fun stopImpl(bySilence: Boolean = false) {
silence.removeCallbacks(silenceWatchdog)
val client = asr
if (audio.isStreaming()) {
audio.stopStreaming()
+ runCatching { client?.endOfTurn(bySilence) }
runCatching { client?.sendAudio(ByteArray(0), isLast = true) }
.onFailure { Log.w(TAG, "isLast failed", it) }
}
- // Wait for 火山 to send the isLast response before closing — otherwise
- // the final transcript never arrives (server sends it ~500ms after our
- // isLast, but a 200ms fixed sleep isn't enough).
- runCatching { client?.waitFinished(3000) }
+ // Wait for the final before closing, or it never arrives. The deadline
+ // is the engine's own — the two need very different ones and neither
+ // is the caller's business; see [AsrEngine.finalWaitMs].
+ runCatching { client?.waitFinished(client.finalWaitMs) }
runCatching { client?.close() }
asr = null
- // Only now, so the media app doesn't resume into 火山's final transcript
- // — that arrives ~500 ms after our isLast and the user is still waiting
- // on it.
+ // Only now, so the media app doesn't resume into the final transcript
+ // — that arrives some time after we said the turn was over, and the
+ // user is still waiting on it.
focus.release()
+ // The band comes down here as well as in `onFinal`, because the paths
+ // that never reach it still leave the microphone's own line on screen.
+ // The tap's acknowledgement is now drawn *before* the connect rather
+ // than after it, so a handshake that fails would otherwise strand
+ // 「听着…」 on the card for good — the turn is over and there is no
+ // engine left to report anything.
+ window.clearSubtitle()
window.setState(FloatingWindowUi.State.IDLE)
}
companion object {
private const val TAG = "Asr"
- /** Quiet this long after the last word and the turn is over. */
- private const val SILENCE_MS = 3000L
+ /**
+ * The longest a single turn may run, whatever the recogniser says.
+ *
+ * A backstop rather than a design. Sixty seconds is far beyond any
+ * question this device is asked and short enough that a failure stays
+ * diagnosable rather than looking like a hang — which is exactly what
+ * it was for twice tonight, when a microphone that would not close read
+ * to the user as a device that would not answer.
+ */
+ private const val MAX_TURN_MS = 60_000L
+
+ /**
+ * How long a turn may run having heard nothing at all before the
+ * silence rule is allowed to end it. See [checkSilence].
+ *
+ * Four seconds, against a measured connection time of 1.4–1.9 s: long
+ * enough to survive a slow handshake and a slow first result, short
+ * enough that a tap followed by silence still ends on its own rather
+ * than waiting out [MAX_TURN_MS].
+ */
+ private const val SILENT_START_GRACE_MS = 4000L
/** How often the watchdog looks; only affects how promptly it stops. */
private const val SILENCE_CHECK_MS = 250L
diff --git a/app/src/main/java/net/kuafuai/andee/asr/AsrEngine.kt b/app/src/main/java/net/kuafuai/andee/asr/AsrEngine.kt
new file mode 100644
index 0000000..51092c0
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/asr/AsrEngine.kt
@@ -0,0 +1,132 @@
+package net.kuafuai.andee.asr
+
+/**
+ * One speech-recognition session, whichever backend is behind it.
+ *
+ * The shape is 火山's, because 火山 was here first: open a session, push PCM
+ * into it as it arrives, ask it to finish, wait for the final line. Both
+ * backends fit it and neither had to be reshaped to fit the other, which is
+ * the only reason this interface exists rather than a branch inside
+ * [AsrController].
+ *
+ * Both backends are streaming ones — 火山 over its own binary protocol, Google
+ * over the Gemini Live API — which is why they reduce to the same five calls.
+ * **The caller's silence rule reads [Listener.onPartial] and nothing else**,
+ * which is what makes the two engines interchangeable rather than merely
+ * similar: both keep reporting while the user talks, so a window that exceeds
+ * their reporting cadence needs to know nothing about the room, the microphone
+ * or which vendor is behind it.
+ */
+interface AsrEngine {
+
+ /**
+ * How long [waitFinished] should be given — **this engine's own doing
+ * included**.
+ *
+ * Asked of the engine rather than fixed by the caller, because the two
+ * differ by more than a fudge factor and the reason is structural. 火山
+ * produces a final *only* in response to being told the utterance is over,
+ * so its answer is on its way as soon as the question is asked and half a
+ * second covers it. The Gemini Live API ends turns on its own voice
+ * detector, so by the time it is asked it has usually already answered and
+ * the request is what provokes a last push of trailing silence plus the
+ * wait for its reply — see `GoogleAsr.finishTurn`.
+ *
+ * One shared number therefore cannot be right for both, and the earlier
+ * version of this was exactly that mistake in reverse: raising the shared
+ * value to suit Google quietly made 火山 slower to give up on a server that
+ * had stopped answering.
+ */
+ val finalWaitMs: Long
+
+ /**
+ * How long a silence the caller should let pass before ending the turn
+ * itself.
+ *
+ * **Two meanings, and which one applies depends on the engine.** For one
+ * that never announces an end of turn this *is* what ends the turn — 火山
+ * has no such signal, so a pause is all there is. For one that does
+ * (the Live API reports `turnComplete` from its own voice detector), this
+ * is the backstop for when that report does not arrive, and the only thing
+ * that closes a microphone on a connection that has gone quiet.
+ *
+ * Per-engine for the same reason as [finalWaitMs]: a single shared value
+ * cannot be right for both, and changing it for one silently changes the
+ * other.
+ */
+ val silenceMs: Long
+
+ interface Listener {
+ /**
+ * The engine is connected and can hear.
+ *
+ * **The caller's silence clock must not run before this.** The
+ * microphone opens at one moment and the recogniser's session comes up
+ * at another, and on this device they are 1.4–1.9 s apart — against a
+ * silence window of 1.5 s. With the clock counting from the microphone,
+ * the deadline landed on top of the handshake, and four measured turns
+ * were cut off at 1.5 s with the first result arriving 100–350 ms later.
+ * Whether any given turn survived came down to tens of milliseconds of
+ * connection luck.
+ *
+ * It does not cover for a broken engine: audio is still captured and
+ * buffered from the moment the microphone opens, and one that never
+ * connects reports [onError] and the turn ends that way.
+ */
+ fun onReady() {}
+
+ /**
+ * What has been heard so far, good enough to draw but not to act on.
+ *
+ * **Both backends call this while the user is still talking**, and
+ * `AsrController` depends on it: the silence watchdog that closes the
+ * microphone is fed from these calls and from nothing else.
+ */
+ fun onPartial(text: String)
+
+ /** The finished transcript. Empty means "there was nothing to hear". */
+ fun onFinal(text: String)
+
+ fun onError(msg: String)
+
+ /** The upstream connection ended. */
+ fun onClosed()
+ }
+
+ /**
+ * The turn is ending, and **how** it is ending.
+ *
+ * [bySilence] is true when the caller's own silence rule expired — see
+ * [silenceMs] — and false when the user ended it by hand or something else
+ * did. The two are not interchangeable to an engine: a silence-ended turn
+ * has, by construction, already been quiet for [silenceMs], which is far
+ * longer than any recogniser needs to settle what it heard, so the last
+ * results are already in hand. A hand-ended turn has no such guarantee —
+ * the user may have stopped mid-word a moment ago.
+ *
+ * Optional to implement: an engine with nothing to gain ignores it, which
+ * is why it carries a default body.
+ */
+ fun endOfTurn(bySilence: Boolean) = Unit
+
+ fun start(sampleRate: Int, channels: Int)
+
+ /**
+ * One block of 16-bit little-endian PCM, or the end-of-utterance marker
+ * when [isLast] — in which case [pcm] is ignored.
+ *
+ * Called on the capture thread; must not block.
+ */
+ fun sendAudio(pcm: ByteArray, isLast: Boolean)
+
+ /**
+ * Block until the final line has been delivered, or [timeoutMs] passes.
+ *
+ * The timeout is the caller's because the two backends need very different
+ * ones: 火山 answers in ~500 ms, while a Google request carries the whole
+ * utterance and has a network round trip of its own.
+ */
+ fun waitFinished(timeoutMs: Long): Boolean
+
+ fun close()
+}
diff --git a/app/src/main/java/net/kuafuai/andee/asr/GoogleAsr.kt b/app/src/main/java/net/kuafuai/andee/asr/GoogleAsr.kt
new file mode 100644
index 0000000..2e8dbb5
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/asr/GoogleAsr.kt
@@ -0,0 +1,710 @@
+package net.kuafuai.andee.asr
+
+import android.util.Base64
+import android.util.Log
+import net.kuafuai.andee.config.VoiceConfig
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
+import okhttp3.WebSocket
+import okhttp3.WebSocketListener
+import okio.ByteString
+import org.json.JSONArray
+import org.json.JSONObject
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+
+/**
+ * Google speech recognition, over the **Gemini Live API** — a WebSocket that
+ * takes audio in and hands text back while the user is still talking.
+ *
+ * **Why a socket, when the rest of the Google path here is plain HTTP.** The
+ * only models Google serves for transcription are the `*-transcribe-live-*`
+ * ones, and they exist on the Live API and nowhere else — the same id on
+ * `generateContent` is a 400. Using a real transcription model and not using a
+ * WebSocket are therefore not both on offer, and this build takes the model: a
+ * general Gemini would have to be *instructed* to transcribe verbatim and
+ * could decide to paraphrase, on a transcript the meeting recorder writes to
+ * disk as the record of what was said.
+ *
+ * **The shape of the conversation is the reference gateway's**, and three
+ * parts of it are things it measured rather than guessed:
+ *
+ * * **The model must be named by its full resource path.** A bare name is
+ * read as a project id → `1008 Permission denied on resource project …`;
+ * `models/x` is a 1007. Hence [VoiceConfig.googleProject] and
+ * [VoiceConfig.googleLocation].
+ * * **`responseModalities` must be `TEXT`.** A transcription model wants the
+ * opposite of a talking one, and getting it backwards drops the connection.
+ * * **`languageCodes` is sent as an explicit empty array**, not omitted, when
+ * no language is set. Omitting the field is not stating a preference;
+ * an empty array is. The two look identical until the upstream changes.
+ *
+ * **Two results per utterance, and only one of them ends the turn.** The
+ * upstream finals *at every pause* — "查一下……杭州的天气" arrives in two
+ * pieces — so its finals are accumulated and shown as [AsrEngine.Listener]
+ * `onPartial`, never reported as the finished transcript. The turn ends when
+ * the *caller* says the user has stopped, which is what `AsrController`'s
+ * three seconds of silence means. Reporting each upstream final as a final
+ * would send half-sentences to the brain at every comma, which is the bug the
+ * reference gateway's accumulator exists to prevent.
+ *
+ * Threading: [sendAudio] is called on the capture thread and returns at once;
+ * all I/O is on [io]; the socket's callbacks arrive on OkHttp's thread.
+ */
+class GoogleAsr(
+ private val config: VoiceConfig,
+ private val listener: AsrEngine.Listener,
+) : AsrEngine {
+
+ /**
+ * [EMIT_GRACE_MS] plus the trailing-silence push and a round trip — see
+ * [finishTurn]. Comfortably more than the engine needs, so that the caller
+ * giving up first is never what loses a transcript.
+ */
+ override val finalWaitMs: Long = EMIT_GRACE_MS + 2000L
+
+ /**
+ * **This is what ends a turn here, the same as for 火山** — see
+ * [VAD_SILENCE_MS] for the experiment that established there is nothing
+ * else. [AsrEngine]'s note explains why the caller can rely on a plain
+ * silence window at all.
+ *
+ * Three seconds, and the number is sized against this engine's own
+ * reporting cadence rather than against a vendor's default. Its results
+ * arrive **about 2.5–4 s after the session is up** — measured over six
+ * turns, during which the caller's watchdog fired on its deadline every
+ * time and the transcripts of 12, 17, 22 and 29 characters were the
+ * opening fraction of what was said. That latency is what the clock has to
+ * survive, and 3 s is the shortest window with room to spare. A published
+ * default like ZEGO's 500 ms is a number for a recogniser that starts
+ * answering immediately.
+ */
+ override val silenceMs: Long = 3000L
+
+ private val io = Executors.newSingleThreadExecutor { r ->
+ Thread(r, "GoogleAsr").apply { isDaemon = true }
+ }
+
+ /**
+ * No read timeout: a Live session is silent for exactly as long as the
+ * user is, and any timeout at all would kill a thoughtful pause. The ping
+ * is what notices a socket that has actually died.
+ */
+ private val http = OkHttpClient.Builder()
+ .connectTimeout(15, TimeUnit.SECONDS)
+ .readTimeout(0, TimeUnit.MILLISECONDS)
+ .pingInterval(20, TimeUnit.SECONDS)
+ .build()
+
+ /** Guards [pending], [finals] and [interim] — written from two threads. */
+ private val lock = Any()
+
+ /** Audio that arrived before the session was up. See [flushPending]. */
+ private val pending = ArrayList<ByteArray>()
+ private var pendingBytes = 0
+
+ /** Settled upstream results, in order. */
+ private val finals = StringBuilder()
+
+ /** The current, still-changing result. Superseded by the next final. */
+ private var interim = ""
+
+ /**
+ * How many **settled** results the upstream sent, and how many hypotheses.
+ *
+ * Kept apart rather than totalled, because the two answer different
+ * questions and an earlier single counter could answer neither. A final is
+ * a sentence the upstream has finished with; an interim is one it is still
+ * revising. Whether the transcript handed to the brain is Google's final
+ * *and* whether a late one is still worth waiting for both depend on the
+ * split, and "results=4" does not say which it was.
+ */
+ @Volatile
+ private var finalsSeen = 0
+
+ @Volatile
+ private var interimsSeen = 0
+
+ /** The last field-set logged; see the note in [handle]. */
+ @Volatile
+ private var lastContentKeys = ""
+
+ /** See [endOfTurn]; decides which grace [finishTurn] uses. */
+ @Volatile
+ private var endedBySilence = false
+
+ @Volatile
+ private var peakMax = 0
+
+ @Volatile
+ private var ws: WebSocket? = null
+
+ @Volatile
+ private var ready = false
+
+ @Volatile
+ private var closed = false
+
+ /** The caller has said the user stopped; an upstream final has no more to add. */
+ private val finished = AtomicBoolean(false)
+
+ /** An upstream final arrived *after* [finished]. See [finishTurn]. */
+ private val lateFinal = CountDownLatch(1)
+
+ private val emitted = AtomicBoolean(false)
+ private val reported = AtomicBoolean(false)
+
+ /** Released when the upstream session is usable, or when it failed. */
+ private val setup = CountDownLatch(1)
+
+ /** Released when the transcript has been delivered. */
+ private val done = CountDownLatch(1)
+
+ override fun start(sampleRate: Int, channels: Int) {
+ Log.d(TAG, "start model=${modelPath()} lang=${config.googleLanguage.ifBlank { "(auto)" }}")
+ io.execute {
+ runCatching { connect() }.onFailure {
+ Log.e(TAG, "connect failed", it)
+ fail(it.message ?: "ASR connect failed")
+ }
+ }
+ }
+
+ /**
+ * Audio while the session is still being set up is **buffered, not
+ * dropped**.
+ *
+ * There is no "I am ready" signal on this side of the handshake: the
+ * capture loop starts as soon as [start] returns, and the setup takes
+ * about a second. What arrives in that second is the beginning of the
+ * user's sentence, and losing it shows up as a connection that works
+ * perfectly and transcribes nothing — with no clue anywhere. The reference
+ * gateway buffers for the same reason and at the same size.
+ */
+ override fun sendAudio(pcm: ByteArray, isLast: Boolean) {
+ if (isLast) {
+ finished.set(true)
+ io.execute { runCatching { finishTurn() } }
+ return
+ }
+ if (closed || pcm.isEmpty()) return
+ peakMax = maxOf(peakMax, pcmPeak(pcm))
+
+ val socket = ws
+ // The decision is made under [lock], and so is the flip to ready, so
+ // that "I buffered this" and "the buffer has been drained" cannot both
+ // be true. Deciding outside the lock leaves a window where a block is
+ // appended just after the drain took its copy — stuck in the list
+ // forever, and audible in the transcript as a dropped word.
+ synchronized(lock) {
+ if (!ready || socket == null) {
+ if (pendingBytes + pcm.size <= MAX_PENDING_BYTES) {
+ pending.add(pcm)
+ pendingBytes += pcm.size
+ }
+ return
+ }
+ // Sent under the lock as well — `send` only enqueues, so the
+ // capture thread loses nothing by it, and it keeps "buffered" and
+ // "sent" from being decided on either side of the drain.
+ socket.send(audioFrame(pcm))
+ }
+ }
+
+ override fun endOfTurn(bySilence: Boolean) {
+ endedBySilence = bySilence
+ }
+
+ override fun waitFinished(timeoutMs: Long): Boolean =
+ done.await(timeoutMs, TimeUnit.MILLISECONDS)
+
+ override fun close() {
+ closed = true
+ // Nobody is left waiting on a socket that is going away.
+ done.countDown()
+ val socket = ws
+ ws = null
+ if (socket != null) {
+ runCatching { socket.close(1000, "done") }.onFailure { socket.cancel() }
+ }
+ io.shutdownNow()
+ }
+
+ // ---- session ----
+
+ private fun connect() {
+ if (config.googleApiKey.isBlank()) {
+ throw IllegalStateException("Google API key 未填写 (⚙ → 后端 → 本机)")
+ }
+
+ val socket = http.newWebSocket(
+ Request.Builder().url("$LIVE_URL?key=${config.googleApiKey}").build(),
+ object : WebSocketListener() {
+ override fun onOpen(webSocket: WebSocket, response: Response) {
+ webSocket.send(setupMessage())
+ }
+
+ override fun onMessage(webSocket: WebSocket, text: String) = handle(text)
+
+ /** The Live API packs its JSON into binary frames, not text ones. */
+ override fun onMessage(webSocket: WebSocket, bytes: ByteString) =
+ handle(bytes.string(Charsets.UTF_8))
+
+ override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
+ // Release the setup wait as *failed*, so connect() does not
+ // sit out its whole timeout on a socket that is already gone.
+ setup.countDown()
+ fail(t.message ?: t.javaClass.simpleName)
+ }
+
+ override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
+ setup.countDown()
+ // A close we did not ask for is a turn that cannot produce
+ // a transcript, so the failure is reported rather than
+ // papered over with whatever has been heard so far: half of
+ // what someone said, handed to the brain as if it were all
+ // of it, is a worse outcome than being told the turn is
+ // dead.
+ if (!closed && !emitted.get()) {
+ Log.w(TAG, "socket closed early code=$code reason=$reason")
+ fail("ASR socket closed ($code)")
+ }
+ }
+ },
+ )
+ ws = socket
+
+ if (!setup.await(SETUP_TIMEOUT_S, TimeUnit.SECONDS)) {
+ throw IllegalStateException("ASR setup 超时 (检查 project number / location / 模型名)")
+ }
+ // The latch is also counted down by a failure, so reaching here is not
+ // proof of success — the error path has already said its piece.
+ if (closed || reported.get()) return
+
+ // Sets `ready` itself, under the lock that [sendAudio] decides in.
+ flushPending()
+ }
+
+ private fun handle(raw: String) {
+ // Frames can still be in flight when the turn ends. A late partial
+ // would repaint the subtitle the controller has already cleared, and
+ // would move a silence clock that is no longer running.
+ if (closed) return
+
+ val d = runCatching { JSONObject(raw) }.getOrNull()
+ if (d == null) {
+ Log.w(TAG, "unparseable frame: ${raw.take(200)}")
+ return
+ }
+
+ if (d.has("setupComplete")) {
+ Log.i(TAG, "session up")
+ setup.countDown()
+ // The session can hear from here — see [AsrEngine.Listener.onReady]
+ // for why the caller needs to be told rather than assuming.
+ listener.onReady()
+ return
+ }
+
+ val error = d.optJSONObject("error")
+ if (error != null) {
+ val msg = error.optString("message").ifBlank { raw.take(200) }
+ setup.countDown()
+ fail("ASR upstream: $msg")
+ return
+ }
+
+ val content = d.optJSONObject("serverContent") ?: return
+
+ // What the upstream actually sends, printed each time the set of fields
+ // changes.
+ //
+ // This is here because a whole design was built on `turnComplete` and
+ // then reverted on the evidence that it never arrives — and "we never
+ // saw it" is not the same claim as "it is not sent". One `serverContent`
+ // frame does not answer it either: the frames differ, an interim
+ // carrying one field and a settled result another. Logging the distinct
+ // shapes of a session settles the question both ways, and costs one
+ // line per shape.
+ val keys = content.keys().asSequence().joinToString()
+ if (keys != lastContentKeys) {
+ lastContentKeys = keys
+ Log.i(TAG, "serverContent fields: $keys")
+ }
+
+ // A hypothesis: shown, not accumulated — it is replaced by the final
+ // for the same speech, and adding it to [finals] would print the
+ // sentence twice.
+ content.optJSONObject("interimInputTranscription")
+ ?.optString("text")?.takeIf { it.isNotEmpty() }
+ ?.let {
+ synchronized(lock) { interim = it }
+ interimsSeen++
+ listener.onPartial(currentText())
+ }
+
+ // A settled result. It supersedes the interim it grew out of, which is
+ // why that is cleared here rather than counted as well.
+ content.optJSONObject("inputTranscription")
+ ?.optString("text")?.takeIf { it.isNotEmpty() }
+ ?.let {
+ synchronized(lock) {
+ finals.append(it)
+ interim = ""
+ }
+ finalsSeen++
+ listener.onPartial(currentText())
+ if (finished.get()) lateFinal.countDown()
+ }
+
+ // The upstream's own end of turn, and it is the primary one from here
+ // on — see [endTurnUpstream].
+ if (content.optBoolean("turnComplete")) endTurnUpstream()
+ }
+
+ /**
+ * The upstream has decided the user finished, and said so.
+ *
+ * **This is what ends a turn now, rather than the caller's silence
+ * timer.** The reasoning is that the upstream is better at the question
+ * than anything measurable here: its detector is trained on speech, and
+ * the local one is a root-mean-square against a tracked noise floor —
+ * which is why three turns tonight opened on a room's noise and
+ * transcribed nothing. It also arrives sooner, because the caller no
+ * longer has to wait out a pause it cannot distinguish from a thought.
+ *
+ * The silence timer has not gone away; it is the backstop for this not
+ * arriving, and [silenceMs] says why it is the longer of the two.
+ *
+ * Delivery goes through the caller's `onFinal` — the same door everything
+ * else uses, so the subtitle, the history row and the brain see one
+ * unchanged shape. `emitted` is what stops [finishTurn] from delivering it
+ * a second time if the caller's own timer happens to fire as well.
+ */
+ private fun endTurnUpstream() {
+ if (closed) return
+ if (!emitted.compareAndSet(false, true)) return
+ val text = currentText()
+ Log.i(
+ TAG,
+ "upstream ended the turn — ${text.length} chars " +
+ "(finals=$finalsSeen interims=$interimsSeen)",
+ )
+ listener.onFinal(text)
+ done.countDown()
+ }
+
+ /**
+ * End the turn: tell the upstream the user stopped, then hand over
+ * everything heard.
+ *
+ * **The trailing silence is not a courtesy, it is what produces the
+ * transcript.** The upstream decides where a turn ends from the *audio
+ * content* — including the quiet parts — so a client that simply stops
+ * sending leaves it with nothing to judge by, and it answers nothing at
+ * all. Measured on this device before the fix: the same code produced
+ * `transcript 12 chars` on one turn and `transcript 0 chars (peak=21799)`
+ * on the next, with the failures correlating exactly with this grace
+ * expiring. [VAD_SILENCE_MS] is the upstream's own threshold, so the push
+ * has to comfortably exceed it, chunked at the same 100 ms granularity as
+ * real capture — one abrupt block of silence is not the same signal as a
+ * stream that trails off.
+ *
+ * `audioStreamEnd` rides along after it, and is exactly the formality the
+ * reference gateway calls it: their measurement is that it does *not* end
+ * the turn on its own (0/3 with it, 0/3 without it, silence is what does).
+ *
+ * **[EMIT_GRACE_MS] must stay well under [finalWaitMs]**, which is what
+ * the caller gives this method to finish in — a caller that has already
+ * given up is one nobody can deliver a transcript to, and the reference
+ * records hitting exactly that: an early fallback "sends the wrong thing,
+ * and the real result arriving afterwards is then thrown away as a
+ * non-final".
+ */
+ private fun finishTurn() {
+ if (closed) return
+ // The upstream already ended this turn and the transcript is delivered.
+ // Pushing silence at a socket that is about to close would be harmless
+ // and pointless, so it is not done.
+ if (emitted.get()) return
+
+ // Decided before the push because the push is what this decides about:
+ // a settled, silence-ended turn does not need one. See the note on the
+ // grace below for why that combination is the safe one — everything
+ // else still gets it, because everything else is a turn where the
+ // upstream has not yet said what it heard.
+ //
+ // **The third condition is the one a long utterance taught.** "Some
+ // final has arrived" is not the same as "everything has been settled":
+ // measured on a 22-second passage, the upstream had delivered exactly
+ // one final and twenty interims, so `finalsSeen > 0` was true while
+ // most of the transcript was still a hypothesis. Taking the fast path
+ // there hands over `finals + interim`, and an interim is by definition
+ // a result the upstream has not finished revising — which is how a long
+ // answer comes back with its tail short. With a pending interim there
+ // is genuinely something still coming, so the full grace is right.
+ val settled = endedBySilence && finalsSeen > 0 &&
+ synchronized(lock) { interim.isEmpty() }
+
+ val socket = ws
+ if (!settled && socket != null && ready) {
+ val silent = ByteArray(SILENCE_CHUNK_BYTES)
+ var pushed = 0
+ while (pushed < TAIL_SILENCE_MS && !closed) {
+ socket.send(audioFrame(silent))
+ pushed += SILENCE_CHUNK_MS
+ }
+ socket.send(JSONObject()
+ .put("realtimeInput", JSONObject().put("audioStreamEnd", true))
+ .toString())
+ Log.d(TAG, "pushed ${TAIL_SILENCE_MS}ms of silence to close the turn")
+ }
+
+ // **Two graces, and the caller's silence rule is what picks between
+ // them.**
+ //
+ // When the turn ended because the caller's own silence window expired,
+ // the upstream has already been fed that silence — three seconds of it
+ // — and its own detector fires at 800 ms. So it settled everything it
+ // heard long before this point, and in five measured turns `finalsSeen`
+ // was never zero here. The remaining wait is spent on nothing, and
+ // three seconds of nothing is the difference between an assistant that
+ // answers and one that pauses first.
+ //
+ // When the user ended the turn by hand, none of that holds: they may
+ // have stopped mid-word a moment ago, the upstream may still be
+ // finalising it, and cutting the wait short would lose the tail of the
+ // sentence they just chose to stop on. That case keeps the full grace.
+ //
+ // `settled` needs all three: the caller's own silence rule ended the
+ // turn, at least one final is in hand, and no hypothesis is still
+ // pending — see its definition above, which is also what skips the
+ // trailing-silence push. Measured five times over, a push into a turn
+ // that already had its final produced nothing at all, so it is not done.
+ val grace = if (settled) SETTLED_GRACE_MS else EMIT_GRACE_MS
+ if (!lateFinal.await(grace, TimeUnit.MILLISECONDS)) {
+ Log.d(
+ TAG,
+ "no late final within ${grace}ms " +
+ "(finals=$finalsSeen bySilence=$endedBySilence) — using what we have",
+ )
+ }
+ emit()
+ }
+
+ private fun emit() {
+ // A turn that already failed does not also get a transcript: reporting
+ // both an error and a final is a contract the caller has no way to
+ // read, and the final it would get would be the empty string anyway.
+ if (reported.get()) return
+ if (!emitted.compareAndSet(false, true)) return
+ val text = currentText()
+ // `results` is here so that "this engine reports late" stays auditable
+ // from a log line rather than trusted: if it is ever
+ // non-zero, the controller's energy VAD is belt-and-braces and the
+ // comment above it needs re-reading.
+ Log.i(
+ TAG,
+ "transcript ${text.length} chars (peak=$peakMax/32767 " +
+ "finals=$finalsSeen interims=$interimsSeen)",
+ )
+ if (closed) {
+ done.countDown()
+ return
+ }
+ listener.onFinal(text)
+ done.countDown()
+ }
+
+ /** Idempotent: a failure that is already reported is not reported twice. */
+ private fun fail(msg: String) {
+ if (!reported.compareAndSet(false, true)) return
+ Log.w(TAG, "failed: $msg")
+ if (!closed) listener.onError(msg)
+ done.countDown()
+ }
+
+ /** Everything settled, plus the hypothesis that has not been settled yet. */
+ private fun currentText(): String = synchronized(lock) {
+ finals.toString() + interim
+ }
+
+ /**
+ * Become ready and send what was held, as one indivisible step.
+ *
+ * [ready] is set *inside* the lock rather than before it: a sender that
+ * sees `ready == true` has then necessarily also seen an empty [pending],
+ * so nothing can be left behind and nothing can overtake. `send` only
+ * enqueues, so doing it while holding the lock costs the capture thread a
+ * moment at most.
+ */
+ private fun flushPending() {
+ synchronized(lock) {
+ ready = true
+ if (pending.isEmpty()) return
+ val socket = ws ?: return
+ Log.i(TAG, "flushing ${pending.size} buffered blocks")
+ for (pcm in pending) socket.send(audioFrame(pcm))
+ pending.clear()
+ pendingBytes = 0
+ }
+ }
+
+ // ---- wire ----
+
+ /**
+ * The setup message, and every field in it is one the reference gateway
+ * arrived at by getting it wrong first — see the class note.
+ *
+ * `silenceDurationMs` is the upstream's own idea of "that pause ends the
+ * turn". It is left at 800 rather than tuned down: the gateway's note on
+ * doing so is that 500 is inside Google's recommended range and still cut
+ * a sentence in half at a breath, and that a parameter changed without a
+ * measurement is the user's experience used as an experiment.
+ */
+ private fun setupMessage(): String {
+ val languages = JSONArray()
+ if (config.googleLanguage.isNotBlank()) languages.put(config.googleLanguage)
+
+ val setup = JSONObject()
+ .put("model", modelPath())
+ .put(
+ "generationConfig",
+ JSONObject().put("responseModalities", JSONArray().put("TEXT")),
+ )
+ .put(
+ "inputAudioTranscription",
+ JSONObject()
+ .put("languageCodes", languages)
+ .put("mode", "VERBATIM"),
+ )
+ .put(
+ "realtimeInputConfig",
+ JSONObject().put(
+ "automaticActivityDetection",
+ JSONObject().put("silenceDurationMs", VAD_SILENCE_MS),
+ ),
+ )
+ return JSONObject().put("setup", setup).toString()
+ }
+
+ private fun audioFrame(pcm: ByteArray): String = JSONObject()
+ .put(
+ "realtimeInput",
+ JSONObject().put(
+ "audio",
+ JSONObject()
+ .put("data", Base64.encodeToString(pcm, Base64.NO_WRAP))
+ .put("mimeType", "audio/pcm;rate=$SAMPLE_RATE"),
+ ),
+ )
+ .toString()
+
+ /**
+ * The full resource path, or the bare name when no project is configured.
+ *
+ * Bare is not a working configuration — the gateway's note is that it comes
+ * back as a permission error naming a project nobody asked for — but
+ * inventing a path around an empty field would replace a clear upstream
+ * error with a confusing local one.
+ */
+ private fun modelPath(): String {
+ val project = config.googleProject
+ if (project.isBlank()) return config.googleAsrModel
+ return "projects/$project/locations/${config.googleLocation}" +
+ "/publishers/google/models/${config.googleAsrModel}"
+ }
+
+ companion object {
+ private const val TAG = "GoogleAsr"
+
+ /**
+ * The Live API endpoint. `v1beta1` is not a typo and not a preference:
+ * it is the version that serves `BidiGenerateContent`.
+ */
+ private const val LIVE_URL =
+ "wss://aiplatform.googleapis.com/ws/" +
+ "google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent"
+
+ private const val SAMPLE_RATE = 16_000
+
+ /** How long the upstream's own silence detector waits before ending a turn. */
+ /**
+ * How long the upstream's voice detector waits before it calls a turn
+ * over. **The reference gateway's measured value, restored after 2500
+ * was tried and reverted.**
+ *
+ * The experiment was to raise this and end the turn on the upstream's
+ * own `turnComplete` instead of on the caller's silence timer, which
+ * would have judged the pause with a trained detector and skipped the
+ * local wait entirely. It does not work here: across every turn
+ * measured the upstream settled its own transcript and **never once
+ * sent `turnComplete`** — so the caller's timer was the only thing that
+ * ever ended a turn, and raising this number only made each of them
+ * twice as long. The field appears to belong to conversational sessions
+ * and not to a transcription-only one like this.
+ *
+ * Back to 800, the gateway's value, and it is kept because it is the
+ * one with a measurement behind it: 500 was tried there and "cut the
+ * sentence in half at a breath".
+ */
+ private const val VAD_SILENCE_MS = 800
+
+ /** Waiting for `setupComplete`. Not getting one means the config is wrong. */
+ private const val SETUP_TIMEOUT_S = 20L
+
+ /**
+ * Silence pushed after the user stops, so the upstream has something to
+ * hear itself stop hearing. Must exceed [VAD_SILENCE_MS] — see
+ * [finishTurn]. The reference gateway's value, kept because it is the
+ * one with a measurement behind it.
+ */
+ private const val TAIL_SILENCE_MS = 1200
+
+ /** One capture block: 100 ms of 16 kHz mono 16-bit. */
+ private const val SILENCE_CHUNK_MS = 100
+ private const val SILENCE_CHUNK_BYTES = SAMPLE_RATE * 2 * SILENCE_CHUNK_MS / 1000
+
+ /**
+ * How long an upstream final is given after the caller says "done".
+ *
+ * **Not the reference gateway's 2000, because that value was measured
+ * on this device and landed on the boundary.** Two consecutive turns:
+ * a final at 1.94 s (transcribed, 16 characters) and one that had not
+ * arrived at 2.00 s (discarded, empty transcript). The cost is real in
+ * both directions — too tight loses the sentence, too loose is silence
+ * in the user's ear — so this sits 50 % above the measured figure and
+ * comfortably under `AsrController.FINAL_WAIT_MS`.
+ */
+ private const val EMIT_GRACE_MS = 3000L
+
+ /**
+ * The wait when the turn ended on the caller's silence rule and a
+ * settled transcript is already in hand — see [finishTurn].
+ *
+ * Not zero, because a result can still be in flight by milliseconds,
+ * and not short-of-a-round-trip either: this is a formality, not a
+ * deadline. Its real justification is the precondition.
+ */
+ private const val SETTLED_GRACE_MS = 300L
+
+ /** 16 kHz × 2 bytes × 20 s, the reference gateway's measured buffer size. */
+ private const val MAX_PENDING_BYTES = 600 * 1024
+
+ /** Absolute peak of 16-bit little-endian PCM. For the log, not for a decision. */
+ private fun pcmPeak(pcm: ByteArray): Int {
+ var max = 0
+ var i = 0
+ while (i + 1 < pcm.size) {
+ val v = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8)
+ val abs = if (v >= 0) v else -v
+ if (abs > max) max = abs
+ i += 2
+ }
+ return max
+ }
+ }
+}
diff --git a/app/src/main/java/net/kuafuai/andee/asr/HuoshanAsr.kt b/app/src/main/java/net/kuafuai/andee/asr/HuoshanAsr.kt
index 5b2e7a2..6205003 100644
--- a/app/src/main/java/net/kuafuai/andee/asr/HuoshanAsr.kt
+++ b/app/src/main/java/net/kuafuai/andee/asr/HuoshanAsr.kt
@@ -30,20 +30,50 @@ import javax.net.ssl.SSLContext
*/
class HuoshanAsr(
private val config: VoiceConfig,
- private val listener: Listener,
-) {
- interface Listener {
- fun onPartial(text: String)
- fun onFinal(text: String)
- fun onError(msg: String)
- fun onClosed()
- }
+ private val listener: AsrEngine.Listener,
+) : AsrEngine {
+
+ /**
+ * How long the server is given to answer our final chunk.
+ *
+ * **3000 is thin, and the three measured turns that say so belong next to
+ * it.** The gap from the silence rule firing to the transcript reaching the
+ * brain was 198 ms, 2676 ms and 2660 ms — a 13× spread whose slow end sits
+ * at 89 % of this budget. Past it the turn does not degrade, it *fails*:
+ * the socket closes with the last frame still in flight, `onFinal` never
+ * fires, and a sentence the user definitely said reaches nothing at all.
+ * That is the failure with no cause visible from the outside, so a turn
+ * that does absolutely nothing starts by looking here.
+ *
+ * 5000 was tried and reverted: this is a ceiling rather than a sleep, so it
+ * buys nothing on a server that answers in 200 ms — which is most of them —
+ * and only lengthens the dead case.
+ */
+ override val finalWaitMs: Long = 3000L
+
+ /**
+ * **火山 never says a turn is over — this is the only thing that ends
+ * one.** Its value is therefore the user-facing pause tolerance, and it is
+ * unchanged from what this device has always used.
+ */
+ override val silenceMs: Long = 3000L
private var client: WebSocketClient? = null
// Full client request uses seq=1; audio chunks start at 2 (火山 protocol).
private val seq = AtomicInteger(2)
+ /** Guards [pending] — the capture thread appends, the connect drains. */
+ private val lock = Any()
+
+ /** Audio that arrived before the handshake landed. See [sendAudio]. */
+ private val pending = ArrayList<ByteArray>()
+ private var pendingBytes = 0
+
+ /** Whether the socket may be written to. Set inside [flushPending]. */
+ @Volatile
+ private var ready = false
+
@Volatile
private var lastPartial: String = ""
@@ -77,7 +107,7 @@ class HuoshanAsr(
@Volatile
private var connected = false
- fun start(sampleRate: Int = 16_000, channels: Int = 1) {
+ override fun start(sampleRate: Int, channels: Int) {
val requestId = UUID.randomUUID().toString()
val connectId = UUID.randomUUID().toString()
val headers = mapOf(
@@ -169,25 +199,97 @@ class HuoshanAsr(
},
)
}
+ // Connected and configured, so the session can hear from here. The
+ // caller's silence clock starts on this rather than on the microphone
+ // opening — see [AsrEngine.Listener.onReady].
+ //
+ // The flush comes first, so that "ready" means the session has
+ // everything said so far — not merely that it could start listening.
+ flushPending()
+ listener.onReady()
+ }
+
+ /**
+ * Send one block, or hold it until the handshake lands.
+ *
+ * **Held rather than dropped, and that is new.** The microphone now opens
+ * before the connect begins — see the note in `AsrController.startImpl` —
+ * so roughly two seconds of speech arrives while the socket is still
+ * connecting. [client] is non-null by then but the connection is not open,
+ * so a send at that point throws `WebsocketNotConnectedException` twenty
+ * times a second and the audio goes with it. Those two seconds are the
+ * beginning of the user's sentence, and losing them shows up as a
+ * connection that works perfectly and transcribes half a question.
+ *
+ * The same shape as [GoogleAsr]'s buffer, and for the same reason: its
+ * engine has always opened the microphone first, because its `start`
+ * returns at once instead of blocking on a connect.
+ */
+ override fun sendAudio(pcm: ByteArray, isLast: Boolean) {
+ val c = client
+ // The decision and the drain happen under one lock, so "held this" and
+ // "the buffer has been emptied" cannot both be true. Deciding outside
+ // it leaves a window where a block is appended just after the drain
+ // took its copy — stuck in the list forever, and audible in the
+ // transcript as a dropped word.
+ synchronized(lock) {
+ if (ready && c != null) {
+ sendNow(c, pcm, isLast)
+ } else if (!isLast && pendingBytes + pcm.size <= MAX_PENDING_BYTES) {
+ // A last frame is never held: it only arrives while the turn is
+ // being torn down, and a session that never came up has nothing
+ // left to finalise.
+ pending.add(pcm)
+ pendingBytes += pcm.size
+ }
+ }
}
- fun sendAudio(pcm: ByteArray, isLast: Boolean) {
- val c = client ?: return
+ /** [lock] held. Assigns the sequence number at *send* time, so order survives buffering. */
+ private fun sendNow(c: WebSocketClient, pcm: ByteArray, isLast: Boolean) {
val current = seq.getAndIncrement()
- val frame = buildAudioRequest(current, pcm, isLast)
- c.send(frame)
+ c.send(buildAudioRequest(current, pcm, isLast))
val n = chunksSent.incrementAndGet()
if (isLast || n <= 3 || n % 20 == 0) {
Log.d(TAG, "sendAudio seq=$current bytes=${pcm.size} isLast=$isLast totalChunks=$n")
}
}
- fun close() {
+ /**
+ * Become ready and hand over everything held, as one indivisible step.
+ *
+ * [ready] is set *inside* the lock rather than before it, for the reason
+ * [GoogleAsr.flushPending] gives: a sender that sees `ready == true` has
+ * then necessarily also seen the drain's effect.
+ */
+ private fun flushPending() {
+ synchronized(lock) {
+ ready = true
+ val c = client
+ if (c != null && pending.isNotEmpty()) {
+ Log.i(TAG, "flushing ${pending.size} buffered blocks")
+ for (pcm in pending) sendNow(c, pcm, isLast = false)
+ pending.clear()
+ pendingBytes = 0
+ }
+ }
+ }
+
+ override fun close() {
try {
client?.close()
} catch (_: Throwable) {
}
client = null
+ // Nothing is left holding audio for a socket that is gone. The capture
+ // has already stopped by the time this runs — [AsrController.stopImpl]
+ // stops it before it closes the engine — so this is a clear, not a
+ // race being won.
+ synchronized(lock) {
+ ready = false
+ pending.clear()
+ pendingBytes = 0
+ }
}
// ---- Response handling ----
@@ -297,7 +399,7 @@ class HuoshanAsr(
* error / socket close). Call this after sending the isLast audio chunk,
* before [close], so the final transcript has a chance to arrive.
*/
- fun waitFinished(timeoutMs: Long = 3000): Boolean =
+ override fun waitFinished(timeoutMs: Long): Boolean =
finishedLatch.await(timeoutMs, TimeUnit.MILLISECONDS)
// ---- Frame builders ----
@@ -358,5 +460,13 @@ class HuoshanAsr(
private const val FLAG_NEG_WITH_SEQUENCE = 0b0011
private const val SERIALIZATION_JSON = 0b0001
private const val COMPRESSION_GZIP = 0b0001
+
+ /**
+ * The hold's ceiling while the handshake is in flight — 16 kHz × 2
+ * bytes × 20 s, the same number [GoogleAsr] uses. Two seconds of
+ * speech is 64 KB; this is the backstop for a microphone that somehow
+ * outruns a connect that never lands.
+ */
+ private const val MAX_PENDING_BYTES = 600 * 1024
}
}
diff --git a/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt b/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
index bc5723d..c51d5c2 100644
--- a/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
+++ b/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
@@ -14,6 +14,40 @@ data class VoiceConfig(
val ttsResourceId: String,
val ttsSpeaker: String,
val ttsSampleRate: Int,
+ /**
+ * [VOICE_HUOSHAN] or [VOICE_GOOGLE] — which service hears and speaks.
+ *
+ * Read here rather than at the call site so that the two engines, the
+ * meeting recorder and the start-up log cannot disagree about it, the same
+ * reason [ttsSpeaker] lives in this object.
+ */
+ val voiceProvider: String,
+ /** Vertex Express key. Empty means the Google path cannot run. */
+ val googleApiKey: String,
+ val googleBaseUrl: String,
+ /**
+ * GCP project **number**, and the region the Live API session is opened
+ * in. Both are only used by the listening half.
+ *
+ * The model has to be named by its full resource path —
+ * `projects/{n}/locations/{r}/publishers/google/models/{m}` — because the
+ * Live API reads a bare name as a project id and answers
+ * `1008 Permission denied on resource project …`. The reference gateway
+ * found that out the hard way; these two rows are what that costs.
+ */
+ val googleProject: String,
+ val googleLocation: String,
+ val googleAsrModel: String,
+ val googleTtsModel: String,
+ /**
+ * A hint for both directions, or blank for "let Google decide".
+ *
+ * Blank rather than a fixed `zh-CN`, following the reference gateway: a
+ * Chinese user says "hello 帮我查一下" and an English one says
+ * "帮我查杭州天气", so naming one language does not restrict the other, it
+ * only makes it less accurate.
+ */
+ val googleLanguage: String,
) {
companion object {
/**
@@ -49,7 +83,14 @@ data class VoiceConfig(
* Keys whose blank value means "go back to the compiled default"
* rather than "the user didn't touch this row". See [save].
*/
- internal val OVERRIDE_KEYS = setOf("api_key", "asr_endpoint", "tts_endpoint")
+ internal val OVERRIDE_KEYS = setOf(
+ "api_key", "asr_endpoint", "tts_endpoint",
+ // The Google rows all have a real value to fall back to — the
+ // build's, or the constant — so an empty box means "use that",
+ // which is the only way back to a baked-in key.
+ "google_api_key", "google_project", "google_base_url", "google_location",
+ "google_asr_model", "google_tts_model", "google_language",
+ )
const val DEFAULT_ASR_ENDPOINT = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
const val DEFAULT_ASR_RESOURCE = "volc.bigasr.sauc.duration"
@@ -58,6 +99,104 @@ data class VoiceConfig(
const val DEFAULT_TTS_SAMPLE_RATE = 24_000
private const val PREFS = "voice_prefs"
+ /**
+ * Which service hears and speaks. See [voiceProvider].
+ *
+ * Two values and not a boolean because the list will not stay at two:
+ * the picker, the self-check rows and the engines all branch on the
+ * string, so a third provider is a value rather than a second flag.
+ */
+ const val VOICE_HUOSHAN = "huoshan"
+ const val VOICE_GOOGLE = "google"
+
+ /**
+ * The Vertex endpoint, its Express-keyed surface.
+ *
+ * Not a region-scoped host: Express mode takes an API key and no
+ * project or location, which is what makes the Google path a single
+ * settings row instead of the five the reference gateway needs.
+ */
+ const val DEFAULT_GOOGLE_BASE_URL = "https://aiplatform.googleapis.com/v1"
+
+ /**
+ * The region the Live API session is opened in.
+ *
+ * The reference gateway measured both this and `global` handshaking
+ * successfully; a real region is the default because it is the one that
+ * cannot silently route a device somewhere it has no business being.
+ */
+ const val DEFAULT_GOOGLE_LOCATION = "asia-northeast1"
+
+ /**
+ * The transcription model, and **the `-live` in its name is the whole
+ * reason the Google listening path is a WebSocket**.
+ *
+ * Google's dedicated transcription models are served only by the Live
+ * API; the same id on `generateContent` is a 400 "is not supported in
+ * the generateContent API". So "no WebSocket" and "a real transcription
+ * model" are not both available, and this build takes the model — a
+ * general Gemini would have to be *told* to transcribe verbatim and
+ * could decide otherwise, on a transcript the meeting recorder writes
+ * to disk as the record of what was said.
+ *
+ * The `-preview` suffix is not optional either: without it the name is
+ * a 404. Both facts are the reference gateway's, measured.
+ */
+ const val DEFAULT_GOOGLE_ASR_MODEL = "gemini-3.5-transcribe-live-preview"
+
+ /**
+ * The speaking model.
+ *
+ * The reference gateway records why this exact id and not the newer
+ * one: `3.8-flash-tts` exists on Gemini's own developer API but not on
+ * Vertex, where the speech models are 3.1 Flash TTS and below, and
+ * asking for 3.8 is a plain "model not found".
+ */
+ const val DEFAULT_GOOGLE_TTS_MODEL = "gemini-3.1-flash-tts-preview"
+
+ /**
+ * What the Google rows resolve to on a device where the user has typed
+ * nothing: **the build's value if it carries one, otherwise the
+ * constant above**.
+ *
+ * This is [llmModelDefault]'s shape for [llmModelDefault]'s reason. The
+ * hint under each row *is* the value a blank resolves to, so the hint
+ * and the resolved config have to come from the same call — a card that
+ * advertises one model and silently calls another is lying in the one
+ * place a user reads to decide what to type.
+ *
+ * The values come from `local.properties` through `BuildConfig` and
+ * never from this file. A key committed to a repository is a key in a
+ * search index, and a bare Vertex key is not a shape secret scanning
+ * recognises — which is exactly why [API_KEY] ships empty and why this
+ * one does too. Whoever builds an APK for Google puts the key in their
+ * own `local.properties`, which `.gitignore` excludes; whoever clones
+ * the repository gets empty strings here and the constants below.
+ *
+ * A build with [googleApiKeyDefault] non-empty is also what makes 谷歌
+ * the default provider — see [voiceProvider].
+ */
+ fun googleApiKeyDefault(): String = BuildConfig.GOOGLE_VOICE_KEY.trim()
+
+ /** See [googleApiKeyDefault]. Empty means "no project baked in". */
+ fun googleProjectDefault(): String = BuildConfig.GOOGLE_VOICE_PROJECT.trim()
+
+ /** See [googleApiKeyDefault]. */
+ fun googleLocationDefault(): String =
+ BuildConfig.GOOGLE_VOICE_LOCATION.trim().ifEmpty { DEFAULT_GOOGLE_LOCATION }
+
+ /** See [googleApiKeyDefault]. */
+ fun googleBaseUrlDefault(): String =
+ BuildConfig.GOOGLE_VOICE_BASE_URL.trim().trimEnd('/').ifEmpty { DEFAULT_GOOGLE_BASE_URL }
+
+ /** See [googleApiKeyDefault]. */
+ fun googleAsrModelDefault(): String =
+ BuildConfig.GOOGLE_VOICE_ASR_MODEL.trim().ifEmpty { DEFAULT_GOOGLE_ASR_MODEL }
+
+ /** See [googleApiKeyDefault]. */
+ fun googleTtsModelDefault(): String =
+ BuildConfig.GOOGLE_VOICE_TTS_MODEL.trim().ifEmpty { DEFAULT_GOOGLE_TTS_MODEL }
+
/**
* Where the agent loop runs. Deliberately an explicit setting rather
* than "hub_url is blank ⇒ go local": [save] drops blank values, so a
@@ -258,7 +397,26 @@ data class VoiceConfig(
*/
fun brainMode(context: Context): String {
val p = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE)
- val saved = p.getString("brain", "")?.trim()?.lowercase().orEmpty()
+ return resolveBrainMode(p.getString("brain", ""))
+ }
+
+ /**
+ * [raw] — a saved value, or one the user has just picked and **not yet
+ * saved** — put through the rule above.
+ *
+ * Split out because the settings card has to ask about the second
+ * kind. The brain picker does not write on tap, so when the user
+ * selects 本机 out of CodeFlying the card is holding a value no pref
+ * reflects yet, and asking [brainMode] there answers about the mode
+ * being left — which is how the voice block came to stay hidden until
+ * 保存 rebuilt the card.
+ *
+ * **One rule, one place.** A second copy of "is CodeFlying really in
+ * play" is how a build that does not enable it ends up hiding the very
+ * voice settings it is reading from.
+ */
+ fun resolveBrainMode(raw: String?): String {
+ val saved = raw?.trim()?.lowercase().orEmpty()
return when (saved) {
BRAIN_HUB -> BRAIN_HUB
BRAIN_LOCAL -> BRAIN_LOCAL
@@ -384,6 +542,47 @@ data class VoiceConfig(
}
}
+ /**
+ * Which service hears and speaks: [VOICE_HUOSHAN] or [VOICE_GOOGLE].
+ *
+ * **Anything that is not exactly `google` reads as 火山**, which is the
+ * same fail-closed direction [groundingEnabled] and [notifyScope] take,
+ * and here it is also the backward-compatible one: every device that
+ * predates this setting has no value at all, and 火山 is what they have
+ * been running. A device that upgrades must not change voice because a
+ * key it never had is now missing.
+ *
+ * **CodeFlying is pinned to 火山, and that is the point of it.**
+ * CodeFlying is the one-tap mode: the APK is packaged by an operator and
+ * handed to someone who configures nothing, so [load] takes every
+ * endpoint *and* credential from the build and the whole errand is the
+ * gateway's. Which means the gateway is also where the choice of
+ * upstream lives — its 火山 support and its Google support are two
+ * handlers behind one URL, and a client that second-guessed that by
+ * dialling Google itself with a key baked into the APK would be
+ * choosing a different vendor and a different bill than the operator
+ * did. So the saved value is ignored here, not merely outranked.
+ *
+ * On any other backend the picker is the user's: their key, their
+ * choice, and a saved value outranks the build's default.
+ */
+ fun voiceProvider(context: Context): String {
+ if (brainMode(context) == BRAIN_CODEFLYING) return VOICE_HUOSHAN
+ val saved = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE)
+ .getString("voice_provider", "").orEmpty().trim().lowercase()
+ return when (saved) {
+ VOICE_GOOGLE -> VOICE_GOOGLE
+ VOICE_HUOSHAN -> VOICE_HUOSHAN
+ // Never chosen. A build packaged with a Google key is a build
+ // packaged *for* Google, so that is what it reads as; a build
+ // without one reads as 火山, which is what every build did
+ // before this setting existed. Either way the picker is one tap
+ // away, and a user's own choice always outranks this.
+ "" -> if (googleApiKeyDefault().isNotEmpty()) VOICE_GOOGLE else VOICE_HUOSHAN
+ else -> VOICE_HUOSHAN
+ }
+ }
+
/**
* Is the 火山 account the one saved in settings, or the factory one?
*
@@ -422,9 +621,35 @@ data class VoiceConfig(
?: DEFAULT_TTS_ENDPOINT,
ttsResourceId = p.getString("tts_resource_id", DEFAULT_TTS_RESOURCE)!!,
// The voice follows the ball's look; [TTS_SPEAKER] is the
- // fallback for looks that don't name one.
+ // fallback for looks that don't name one. The same speaker
+ // string is what the Google path maps to a Gemini voice, so
+ // this stays the one place a voice is decided.
ttsSpeaker = lookSpeaker(p),
ttsSampleRate = p.getInt("tts_sample_rate", DEFAULT_TTS_SAMPLE_RATE),
+ voiceProvider = voiceProvider(context),
+ // Every one of these is saved-value-else-build-default, and the
+ // card's hint under each row is the same call — see
+ // [googleApiKeyDefault]. On a build with nothing baked in that
+ // reduces to exactly what it was before: the constants.
+ googleApiKey = p.getString("google_api_key", "").orEmpty().trim()
+ .ifEmpty { googleApiKeyDefault() },
+ googleBaseUrl = p.getString("google_base_url", "").orEmpty().trim()
+ .trimEnd('/').ifEmpty { googleBaseUrlDefault() },
+ // The project number has no constant to fall back to — there is
+ // no sensible one to invent, and the failure a missing one
+ // produces is a 1008 naming the resource, which says what is
+ // wrong better than an invented value could.
+ googleProject = p.getString("google_project", "").orEmpty().trim()
+ .ifEmpty { googleProjectDefault() },
+ googleLocation = p.getString("google_location", "").orEmpty().trim()
+ .ifEmpty { googleLocationDefault() },
+ googleAsrModel = p.getString("google_asr_model", "").orEmpty().trim()
+ .ifEmpty { googleAsrModelDefault() },
+ googleTtsModel = p.getString("google_tts_model", "").orEmpty().trim()
+ .ifEmpty { googleTtsModelDefault() },
+ // Blank is a real value here, not a missing one — see the
+ // field's own note. It must not fall through to a default.
+ googleLanguage = p.getString("google_language", "").orEmpty().trim(),
)
}
@@ -471,6 +696,19 @@ data class VoiceConfig(
"tts_endpoint" to p.getString("tts_endpoint", "").orEmpty(),
"tts_resource_id" to p.getString("tts_resource_id", DEFAULT_TTS_RESOURCE)!!,
"tts_sample_rate" to p.getInt("tts_sample_rate", DEFAULT_TTS_SAMPLE_RATE).toString(),
+ // Which service hears and speaks, plus its rows. The Google
+ // endpoint/model rows show saved-or-empty for the same reason
+ // the 火山 ones do: an empty box is the only state in which
+ // the hint — which is the value a blank resolves to — can be
+ // seen at all.
+ "voice_provider" to voiceProvider(context),
+ "google_api_key" to p.getString("google_api_key", "").orEmpty(),
+ "google_base_url" to p.getString("google_base_url", "").orEmpty(),
+ "google_project" to p.getString("google_project", "").orEmpty(),
+ "google_location" to p.getString("google_location", "").orEmpty(),
+ "google_asr_model" to p.getString("google_asr_model", "").orEmpty(),
+ "google_tts_model" to p.getString("google_tts_model", "").orEmpty(),
+ "google_language" to p.getString("google_language", "").orEmpty(),
"hub_url" to hc.url,
"device_id" to hc.deviceId,
"device_name" to hc.deviceName,
@@ -711,6 +949,10 @@ data class VoiceConfig(
"api_key",
"asr_endpoint", "asr_resource_id",
"tts_endpoint", "tts_resource_id", "tts_sample_rate",
+ "voice_provider",
+ "google_api_key", "google_base_url",
+ "google_project", "google_location",
+ "google_asr_model", "google_tts_model", "google_language",
"hub_url", "device_name",
"brain",
"show_hub",
diff --git a/app/src/main/java/net/kuafuai/andee/device/SelfCheck.kt b/app/src/main/java/net/kuafuai/andee/device/SelfCheck.kt
index 9d42d6c..4c2ba6b 100644
--- a/app/src/main/java/net/kuafuai/andee/device/SelfCheck.kt
+++ b/app/src/main/java/net/kuafuai/andee/device/SelfCheck.kt
@@ -244,7 +244,7 @@ object SelfCheck {
adbKeyboard(context, a11yIme),
secureSettings(context, a11yIme),
voiceKey(context),
- voiceReach(probe, online),
+ voiceReach(context, probe, online),
wakePhrase(context),
background(context),
storage(context),
@@ -862,22 +862,35 @@ object SelfCheck {
}
/**
- * Is there a 火山 key at all.
+ * Is there a key for the voice service that is actually selected.
*
* Free, offline, and it sits *before* [voiceReach] because it answers the
* cheaper half of the same question. Without a key ASR and TTS fail during
- * the WebSocket handshake, which surfaces as a connect error — the device
- * looks like it has a network problem when what it has is an empty field.
+ * their handshake, which surfaces as a connect error — the device looks
+ * like it has a network problem when what it has is an empty field.
* [VoiceConfig.API_KEY] ships empty in the public source (see its doc), so
* this is the state every fresh checkout starts in, and it has to be said
* out loud rather than discovered.
*
+ * **Which key is asked for follows the provider, and the row names
+ * neither.** The checklist is about "can this device hear and speak"; a
+ * green tick earned by the 火山 key while the device is talking to Google
+ * is the exact lie that having one checklist instead of two exists to
+ * prevent. Not naming the provider in the sentence is what keeps [Finding]
+ * language-free — see the note on `detailArgs` — and the settings card
+ * that the fix button opens is showing which one is picked anyway.
+ *
* WARN, not FAIL: the brain, the screen tools and the notebook all work
* mute. What is dead is hearing and speaking, which is a degraded device
* rather than a dead one.
*/
private fun voiceKey(context: Context): Finding {
- val has = VoiceConfig.load(context).apiKey.isNotBlank()
+ val cfg = VoiceConfig.load(context)
+ val has = if (cfg.voiceProvider == VoiceConfig.VOICE_GOOGLE) {
+ cfg.googleApiKey.isNotBlank()
+ } else {
+ cfg.apiKey.isNotBlank()
+ }
return Finding(
id = "voice_key",
title = R.string.check_title_voice_key,
@@ -889,18 +902,28 @@ object SelfCheck {
}
/**
- * Can we reach 火山 at all.
- *
- * A TCP connect, **not** a credential check: the ASR/TTS handshake carries
- * the key in headers, and reproducing it here would mean a second copy of
- * that protocol living next to `HuoshanAsr`. What this catches is the
- * failure this device actually had — `ASR ws connect timeout` — where the
- * network is up and the voice host still is not reachable from it. A green
- * row here beside a failing ASR therefore means the key, and says so by
+ * Can we reach the selected service's host at all.
+ *
+ * A TCP connect, **not** a credential check: both handshakes carry the key
+ * in headers, and reproducing either here would mean a second copy of that
+ * protocol living next to its client. What this catches is the failure
+ * this device actually had — `ASR ws connect timeout` — where the network
+ * is up and the voice host still is not reachable from it. A green row
+ * here beside a failing ASR therefore means the key, and says so by
* elimination — a deduction that is only usable because [voiceKey] has
* already ruled out the empty field.
+ *
+ * The host moves with the provider, because the deduction above only holds
+ * if the row is probing the service in use. The Google endpoint is the
+ * inference host both halves go through — recognition opens a socket to
+ * it, synthesis posts to it.
*/
- private fun voiceReach(probe: Boolean, online: Boolean): Finding {
+ private fun voiceReach(context: Context, probe: Boolean, online: Boolean): Finding {
+ val host = if (VoiceConfig.voiceProvider(context) == VoiceConfig.VOICE_GOOGLE) {
+ VOICE_HOST_GOOGLE
+ } else {
+ VOICE_HOST
+ }
if (!probe || !online) {
return Finding(
id = "voice_host",
@@ -910,14 +933,14 @@ object SelfCheck {
else R.string.check_detail_skipped_offline,
)
}
- val err = tcpProbe(VOICE_HOST, VOICE_PORT)
+ val err = tcpProbe(host, VOICE_PORT)
return Finding(
id = "voice_host",
title = R.string.check_title_voice_host,
level = if (err == null) Level.OK else Level.WARN,
detail = if (err == null) R.string.check_detail_voice_host_ok
else R.string.check_detail_voice_host_bad,
- detailArgs = if (err == null) listOf(VOICE_HOST) else listOf(VOICE_HOST, err),
+ detailArgs = if (err == null) listOf(host) else listOf(host, err),
)
}
@@ -1297,6 +1320,16 @@ object SelfCheck {
"https://github.com/senzhk/ADBKeyBoard/releases/download/v2.5-dev/keyboardservice-debug.apk"
private const val VOICE_HOST = "openspeech.bytedance.com"
+
+ /**
+ * Vertex's inference host — where both halves of the Google path go.
+ *
+ * Not the `wss://…/ws/…BidiGenerateContent` path and not a per-region
+ * host: the probe is a TCP connect, so the name that matters is the one
+ * that has to resolve and accept a connection.
+ */
+ private const val VOICE_HOST_GOOGLE = "aiplatform.googleapis.com"
+
private const val VOICE_PORT = 443
private const val PROBE_CONNECT_S = 6L
private const val PROBE_CONNECT_MS = 6000
diff --git a/app/src/main/java/net/kuafuai/andee/meeting/MeetingRecorder.kt b/app/src/main/java/net/kuafuai/andee/meeting/MeetingRecorder.kt
index 0c6f8d9..5ed80df 100644
--- a/app/src/main/java/net/kuafuai/andee/meeting/MeetingRecorder.kt
+++ b/app/src/main/java/net/kuafuai/andee/meeting/MeetingRecorder.kt
@@ -6,6 +6,8 @@ import android.os.Looper
import android.os.SystemClock
import android.util.Log
import net.kuafuai.andee.R
+import net.kuafuai.andee.asr.AsrEngine
+import net.kuafuai.andee.asr.GoogleAsr
import net.kuafuai.andee.asr.HuoshanAsr
import net.kuafuai.andee.audio.AudioFocusGate
import net.kuafuai.andee.audio.AudioIO
@@ -29,20 +31,35 @@ import java.util.concurrent.atomic.AtomicInteger
* screen with `show_html`. That split is why this class has no notion of a
* "summary" anywhere in it.
*
- * **Why segments.** [HuoshanAsr] reports a final transcript exactly once, when
- * the server answers our `isLast`. One session for one meeting would therefore
- * produce nothing at all until the meeting ended, and any dropped connection in
- * between would take the whole hour with it. So the recogniser is rotated: a
- * segment lives ~2 minutes, its final text is appended to disk the moment it
- * arrives, and a disconnect costs one segment instead of the meeting.
+ * **Why segments.** A recogniser reports a final transcript exactly once, when
+ * it is told the utterance is over — and neither engine produces one before
+ * that. One session for one meeting would therefore produce nothing at all
+ * until the meeting ended, and any dropped connection in between would take
+ * the whole hour with it. So the recogniser is rotated: a segment lives ~2
+ * minutes, its final text is appended to disk the moment it arrives, and a
+ * disconnect costs one segment instead of the meeting.
*
* **Why the new session opens before the old one is retired.** Retiring costs
- * up to [FINAL_WAIT_MS] of blocking (火山 sends the final ~500 ms after our
- * `isLast` — see `AsrController.stopImpl`), and connecting costs a second or
- * two. Doing either with no live session would punch a hole in the recording.
+ * up to the engine's own final wait of blocking (a recogniser owes a final once
+ * it has been told the segment is over — see [AsrEngine.finalWaitMs]), and
+ * connecting costs a second or two. Doing either with no live session would
+ * punch a hole in the recording.
* Opening first means audio always has somewhere to go, and it makes a failed
* rotation free: if the replacement cannot connect, the old segment is still
* receiving and the meeting carries on.
+ *
+ * **Why the recogniser is not 火山-specific.** The three properties this class
+ * needs are the three [AsrEngine] already promises: audio goes in as it
+ * arrives, listening continues while it does, and one final comes back when
+ * the caller says the segment is over. 火山 and the Google Live API both have
+ * them, so a segment is built from [VoiceConfig.voiceProvider] like every
+ * other listener on this device. A meeting is not a different kind of
+ * listening, and a device configured only for Google can take minutes.
+ *
+ * The wait in [retire] is provider-dependent, and that is why it is asked of
+ * the engine rather than named here: 火山 answers a retiring segment in about
+ * half a second, while the Google engine pushes trailing silence and waits for
+ * the reply. A single constant in this file could only ever suit one of them.
*/
class MeetingRecorder(
private val context: Context,
@@ -189,7 +206,7 @@ class MeetingRecorder(
.put("file", fileRef())
}
- /** Synchronous stop, for the dispatcher. Blocks for up to [FINAL_WAIT_MS]. */
+ /** Synchronous stop, for the dispatcher. Blocks for the engine's final wait. */
fun stop(): JSONObject {
if (!recording.compareAndSet(true, false)) {
throw IllegalStateException("no meeting is being recorded")
@@ -205,7 +222,7 @@ class MeetingRecorder(
* asked it for one.
*
* Safe from the main thread and from a binder thread: the teardown blocks
- * on 火山's last word, which would ANR the UI, so it goes to [exec] and the
+ * on the recogniser's last word, which would ANR the UI, so it goes to [exec] and the
* result comes back through [onEnded].
*
* @param reason reaches the brain verbatim as `stopped_by`, and it says
@@ -238,8 +255,8 @@ class MeetingRecorder(
*
* Blocks, because the caller releases the microphone the instant this
* returns and the last segment's text exists only in memory until [io]
- * runs — but without [FINAL_WAIT_MS], which would put three seconds into
- * `onDestroy` to gain a final that the settled partial already covers.
+ * runs — but without waiting for the final, which would put seconds into
+ * `onDestroy` to gain text the settled partial already covers.
*/
fun shutdown() {
if (recording.compareAndSet(true, false)) {
@@ -325,7 +342,11 @@ class MeetingRecorder(
private fun retire(seg: Segment, waitForFinal: Boolean = true) {
runCatching { seg.asr.sendAudio(ByteArray(0), isLast = true) }
- if (waitForFinal) runCatching { seg.asr.waitFinished(FINAL_WAIT_MS) }
+ // The deadline is the engine's own: 火山 answers a retiring segment in
+ // about half a second, while the Google engine pushes trailing silence
+ // and waits for the reply. One shared number could only be right for
+ // one of them — see [AsrEngine.finalWaitMs].
+ if (waitForFinal) runCatching { seg.asr.waitFinished(seg.asr.finalWaitMs) }
runCatching { seg.asr.close() }
// The latch also trips on error and on close, so a final is not
// guaranteed to have arrived. Settling here keeps the last partial
@@ -360,7 +381,7 @@ class MeetingRecorder(
val f = file ?: return
// runCatching around the submit too: a segment can settle from its
// WebSocket thread after [shutdown] closed the writer, and a rejected
- // task must not surface as a crash inside 火山's callback.
+ // task must not surface as a crash inside the recogniser's callback.
runCatching {
io.execute {
runCatching { f.appendText(block) }
@@ -455,8 +476,16 @@ class MeetingRecorder(
// ---- One recogniser session ----------------------------------------
- private inner class Segment(val index: Int) : HuoshanAsr.Listener {
- val asr = HuoshanAsr(VoiceConfig.load(context), this)
+ private inner class Segment(val index: Int) : AsrEngine.Listener {
+ // Whichever service hears, the same one the conversation uses — a
+ // meeting is not a different kind of listening. Both engines suit a
+ // segment exactly: audio goes in for as long as the segment lives,
+ // and one final comes back when it is told the segment is over.
+ val asr: AsrEngine = if (VoiceConfig.voiceProvider(context) == VoiceConfig.VOICE_GOOGLE) {
+ GoogleAsr(VoiceConfig.load(context), this)
+ } else {
+ HuoshanAsr(VoiceConfig.load(context), this)
+ }
/** Reset by [rotate] when a replacement could not be opened. */
@Volatile
@@ -523,7 +552,14 @@ class MeetingRecorder(
/** Quiet this long and the seam is safe to put here. */
const val PAUSE_MS = 800L
- /** 火山 sends its final ~500 ms after our isLast; this is the ceiling. */
+ /**
+ * How long a retiring segment waits for its final.
+ *
+ * 火山 answers in about half a second; the Google engine holds the
+ * segment's text for its own grace first (`GoogleAsr.EMIT_GRACE_MS`)
+ * and that has to fit inside this. A caller that has already given up
+ * is one nobody can hand a transcript to.
+ */
const val FINAL_WAIT_MS = 3000L
/** Roughly a four-hour Chinese meeting. Beyond it, see [elide]. */
diff --git a/app/src/main/java/net/kuafuai/andee/tts/GoogleTts.kt b/app/src/main/java/net/kuafuai/andee/tts/GoogleTts.kt
new file mode 100644
index 0000000..987394b
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/tts/GoogleTts.kt
@@ -0,0 +1,368 @@
+package net.kuafuai.andee.tts
+
+import android.util.Base64
+import android.util.Log
+import net.kuafuai.andee.config.VoiceConfig
+import okhttp3.Call
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
+import org.json.JSONArray
+import org.json.JSONObject
+import java.io.BufferedReader
+import java.io.IOException
+import java.io.InputStreamReader
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+
+/**
+ * Google text-to-speech — Gemini's `streamGenerateContent`, over plain HTTP
+ * with an SSE body.
+ *
+ * **The shape of the protocol is the one thing to understand here: whole text
+ * in, audio chunks out.** Gemini TTS does not accept streaming *input*, so
+ * "speaking as the model thinks" has to be arranged by the caller — which is
+ * why [finish] splits the utterance into sentences and synthesises them one
+ * after another. The first sentence's audio is playing while the second is
+ * still being made, and [Listener.onAudio] streams each chunk as it lands.
+ *
+ * Three things here are measured, not chosen:
+ *
+ * * **The endpoint is `streamGenerateContent?alt=sse`, not `generateContent`.**
+ * Both are HTTP; the streaming one is what the reference gateway runs in
+ * production against this exact model, and it starts producing sound
+ * before the sentence is finished.
+ * * **`role: "user"` must be written out.** The reference records that
+ * omitting it is accepted on one Google surface and a
+ * `400 Please use a valid role` on another, and that the cost of always
+ * sending it is one field.
+ * * **`/publishers/google` in the path is not optional** — its absence is a
+ * 404 with an HTML body, which reads as "wrong host" rather than "wrong
+ * path". Same reason the reference's client logs the first raw response
+ * chunk: the audio container is only ever learned from that field.
+ *
+ * See `GoogleAsr` for why this device speaks HTTP to Google instead of using
+ * the WebSocket the vendor's own live models want.
+ */
+class GoogleTts(
+ private val config: VoiceConfig,
+ private val listener: TtsEngine.Listener,
+) : TtsEngine {
+
+ private val text = StringBuilder()
+
+ @Volatile
+ private var cancelled = false
+
+ /** The in-flight request, so [cancel] can cut a long answer short. */
+ @Volatile
+ private var call: Call? = null
+
+ /** Cleared per utterance, so the rate check reports once each time. */
+ @Volatile
+ private var rateChecked = false
+
+ /** Cleared per utterance; see [request]. */
+ @Volatile
+ private var firstChunkLogged = false
+
+ private val io = Executors.newSingleThreadExecutor { r ->
+ Thread(r, "GoogleTts").apply { isDaemon = true }
+ }
+
+ private val http = OkHttpClient.Builder()
+ .connectTimeout(15, TimeUnit.SECONDS)
+ .readTimeout(60, TimeUnit.SECONDS)
+ .writeTimeout(30, TimeUnit.SECONDS)
+ .build()
+
+ /**
+ * The session is up immediately — there is no handshake to wait for.
+ *
+ * **This is what makes the sample rate a declared constant rather than
+ * something read off the wire**, and it is a direct consequence of
+ * [TtsEngine]'s call order: the driver pushes its text from `onOpen`, so
+ * `onOpen` has to come before the request that would tell us the real
+ * rate. Gemini's speech output is L16 at 24 kHz, which is also what the
+ * 火山 speaker reports, so the two backends hand the player the same
+ * number. [checkRate] verifies it against the response and says so loudly
+ * if a future model disagrees.
+ */
+ override fun open() {
+ io.execute {
+ if (cancelled) return@execute
+ listener.onOpen(SAMPLE_RATE)
+ }
+ }
+
+ override fun push(text: String) {
+ synchronized(this) { this.text.append(text) }
+ }
+
+ override fun finish() {
+ val pending = synchronized(this) { this.text.toString() }
+ io.execute { runCatching { synth(pending) }.onFailure {
+ // A cancel surfaces here as an IOException, and a barge-in is a
+ // cancel — so logging it as a failure would put a stack trace in
+ // the log every time the user interrupts the assistant, and send
+ // the owner looking for a fault in a turn that worked.
+ if (cancelled) return@onFailure
+ Log.e(TAG, "synth failed", it)
+ listener.onError(it.message ?: "tts failed")
+ } }
+ }
+
+ override fun cancel() {
+ cancelled = true
+ runCatching { call?.cancel() }
+ }
+
+ private fun synth(pending: String) {
+ if (cancelled) return
+ val sentences = split(pending)
+ if (sentences.isEmpty()) {
+ // Nothing pronounceable — punctuation, or whitespace. The turn is
+ // still over, and saying nothing is the correct reading of it.
+ listener.onDone()
+ return
+ }
+
+ rateChecked = false
+ var sawAny = false
+ for ((i, sentence) in sentences.withIndex()) {
+ if (cancelled) return
+ listener.onSentenceStart(sentence)
+ val got = request(sentence)
+ listener.onSentenceEnd(sentence)
+ if (cancelled) return
+ sawAny = sawAny || got
+ // The first sentence is the test of whether this works at all. A
+ // silent *later* sentence is a long pause and not our business; a
+ // silent *first* one means the whole utterance is about to vanish
+ // into nothing, and reporting it is what gets the words put up on
+ // screen to be read instead.
+ if (i == 0 && !got) {
+ listener.onError("Google TTS returned no audio")
+ return
+ }
+ }
+ if (cancelled) return
+ listener.onDone()
+ }
+
+ /**
+ * One sentence, streamed. Returns whether any audio came back.
+ *
+ * The body is read line by line: the response is SSE (`data: {...}` per
+ * line) and chunk boundaries mean nothing, so a fixed buffer size would
+ * break on a long sentence.
+ */
+ private fun request(sentence: String): Boolean {
+ val url = config.googleBaseUrl.trimEnd('/') +
+ "/publishers/google/models/" + config.googleTtsModel +
+ ":streamGenerateContent?alt=sse&key=" + config.googleApiKey
+
+ val request = Request.Builder()
+ .url(url)
+ .header("Content-Type", "application/json; charset=utf-8")
+ .post(buildBody(sentence).toString().toRequestBody(JSON_MEDIA))
+ .build()
+
+ val c = http.newCall(request)
+ call = c
+ var sawAudio = false
+ try {
+ c.execute().use { resp ->
+ if (!resp.isSuccessful) {
+ val raw = resp.body?.string().orEmpty()
+ throw IOException("TTS HTTP ${resp.code}: ${detail(raw)}")
+ }
+ val body = resp.body ?: throw IOException("TTS returned an empty body")
+
+ BufferedReader(InputStreamReader(body.byteStream(), Charsets.UTF_8)).use { reader ->
+ while (true) {
+ val line = reader.readLine() ?: break
+ if (cancelled) break
+ // The shape of this stream is only ever learned from the
+ // stream. The documentation says "audio" and does not
+ // say whether that is raw PCM, which field it hides in,
+ // or whether `alt=sse` is honoured at all — and the
+ // reference gateway logs its first chunk for exactly
+ // that reason. Skipping that line here cost a debugging
+ // round: "returned no audio" is what a wrong guess about
+ // the shape looks like from the outside.
+ //
+ // Logged *before* the `data:` test on purpose, so a
+ // response that is not SSE at all — the case where every
+ // later log line would be silent — still says something.
+ if (!firstChunkLogged && line.isNotBlank()) {
+ firstChunkLogged = true
+ Log.i(TAG, "first line of response: ${line.take(800)}")
+ }
+ if (!line.startsWith("data:")) continue
+ val payload = line.substring(5).trim()
+ if (payload.isEmpty() || payload == "[DONE]") continue
+
+ val chunk = runCatching { JSONObject(payload) }.getOrNull()
+ if (chunk == null) {
+ Log.w(TAG, "unparseable chunk: ${payload.take(200)}")
+ continue
+ }
+ for (pcm in audioOf(chunk)) {
+ sawAudio = true
+ listener.onAudio(pcm)
+ }
+ }
+ }
+ }
+ } finally {
+ call = null
+ }
+ return sawAudio
+ }
+
+ /**
+ * Every audio part in one SSE chunk, decoded.
+ *
+ * Defensive at every level on purpose: this is a shape taken from another
+ * codebase's working client rather than from a schema, and a `candidates`
+ * array that is absent, empty, or carries a text part instead of audio is
+ * all "no audio in this chunk", not an exception.
+ */
+ private fun audioOf(chunk: JSONObject): List<ByteArray> {
+ val out = ArrayList<ByteArray>()
+ val parts = chunk.optJSONArray("candidates")
+ ?.optJSONObject(0)
+ ?.optJSONObject("content")
+ ?.optJSONArray("parts") ?: return out
+
+ for (i in 0 until parts.length()) {
+ val inline = parts.optJSONObject(i)?.optJSONObject("inlineData") ?: continue
+ val data = inline.optString("data")
+ if (data.isBlank()) continue
+ checkRate(inline.optString("mimeType"))
+ val pcm = runCatching { Base64.decode(data, Base64.DEFAULT) }.getOrNull() ?: continue
+ if (pcm.isNotEmpty()) out.add(pcm)
+ }
+ return out
+ }
+
+ /**
+ * Compare the wire's sample rate against the one we already told the
+ * player, once per utterance.
+ *
+ * This cannot be fixed from here — `TtsEngine`'s call order means the
+ * AudioTrack exists before the first response arrives — so the honest
+ * thing is to say so at the exact moment it becomes knowable, with both
+ * numbers, rather than let someone debug a chipmunk by ear later.
+ */
+ private fun checkRate(mime: String) {
+ if (rateChecked || mime.isBlank()) return
+ rateChecked = true
+ val rate = Regex("rate=(\\d+)").find(mime)?.groupValues?.get(1)?.toIntOrNull() ?: return
+ if (rate != SAMPLE_RATE) {
+ Log.e(TAG, "Gemini TTS returned ${rate}Hz but the player was opened at ${SAMPLE_RATE}Hz ($mime)")
+ }
+ }
+
+ /**
+ * The request body, and every field in it is one the reference gateway
+ * arrived at by getting it wrong first: `responseModalities: ["AUDIO"]`
+ * (a transcription model wants the opposite), `role` written out, and the
+ * voice nested three deep in `prebuiltVoiceConfig`.
+ *
+ * `languageCode` is omitted rather than blanked when unset. The audio is
+ * what decides how to say the words; naming a language is a hint, and an
+ * empty string is not the same message as "no hint". This is the same
+ * distinction the reference makes for ASR.
+ */
+ private fun buildBody(sentence: String): JSONObject {
+ val prebuilt = JSONObject().put("voiceName", voiceFor(config.ttsSpeaker))
+ val speechConfig = JSONObject().put(
+ "voiceConfig",
+ JSONObject().put("prebuiltVoiceConfig", prebuilt),
+ )
+ if (config.googleLanguage.isNotBlank()) {
+ speechConfig.put("languageCode", config.googleLanguage)
+ }
+
+ return JSONObject()
+ .put(
+ "contents",
+ JSONArray().put(
+ JSONObject()
+ .put("role", "user")
+ .put("parts", JSONArray().put(JSONObject().put("text", sentence))),
+ ),
+ )
+ .put(
+ "generationConfig",
+ JSONObject()
+ .put("responseModalities", JSONArray().put("AUDIO"))
+ .put("speechConfig", speechConfig),
+ )
+ }
+
+ /**
+ * Split into sentences, keeping the terminator attached to the sentence it
+ * ends.
+ *
+ * The point is time-to-first-sound, not prosody: one request per sentence
+ * means the first sentence is being heard while the last is still being
+ * synthesised. The cost is that a sentence is synthesised without
+ * knowledge of the next, which is exactly the trade the reference gateway
+ * makes — a client that pushes text sentence by sentence gets a Gemini
+ * call per sentence there too.
+ *
+ * `. ` rather than `.` in the pattern: a bare period would cut `3.5` in
+ * half, and a half-sentence is spoken with a falling contour that sounds
+ * like the end of a thought.
+ */
+ private fun split(pending: String): List<String> =
+ pending.split(Regex("(?<=[。!?!?\\n])|(?<=\\. )"))
+ .map { it.trim() }
+ .filter { it.isNotEmpty() && it.any { ch -> ch.isLetterOrDigit() } }
+
+ /** The API's own words for what it disliked, else a truncated body. */
+ private fun detail(raw: String): String = runCatching {
+ JSONObject(raw).optJSONObject("error")?.optString("message").orEmpty()
+ }.getOrDefault("").ifBlank { raw.take(400) }
+
+ companion object {
+ private const val TAG = "GoogleTts"
+
+ private val JSON_MEDIA = "application/json; charset=utf-8".toMediaType()
+
+ /** Gemini speech output: signed 16-bit little-endian, mono. */
+ private const val SAMPLE_RATE = 24_000
+
+ /** Used when the ball's look names a speaker this build has no mapping for. */
+ const val DEFAULT_VOICE = "Kore"
+
+ /**
+ * The character's voice, in Google's names.
+ *
+ * The rule this preserves is the one the whole voice story rests on —
+ * **the voice is the character's**, there is no voice row for the user
+ * to get wrong — and the mapping is the reference gateway's, chosen by
+ * timbre rather than by personality: personality is already the look's
+ * `persona` text, and picking a voice to match *that* produces a
+ * character whose face and voice disagree.
+ *
+ * An unknown speaker is not an error. A look added later simply speaks
+ * in the default; refusing to talk because a table has no row would be
+ * a worse outcome than a different voice.
+ */
+ private val VOICE_MAP = mapOf(
+ "zh_female_vv_uranus_bigtts" to "Leda",
+ "zh_male_zhubajie_uranus_bigtts" to "Algenib",
+ "zh_male_lubanqihao_uranus_bigtts" to "Fenrir",
+ "zh_female_peiqi_uranus_bigtts" to "Zephyr",
+ "zh_male_xionger_uranus_bigtts" to "Sulafat",
+ )
+
+ /** [VOICE_MAP] with the default for anything it does not know. */
+ fun voiceFor(speaker: String): String = VOICE_MAP[speaker] ?: DEFAULT_VOICE
+ }
+}
diff --git a/app/src/main/java/net/kuafuai/andee/tts/HuoshanTts.kt b/app/src/main/java/net/kuafuai/andee/tts/HuoshanTts.kt
index cacd06e..c9f2438 100644
--- a/app/src/main/java/net/kuafuai/andee/tts/HuoshanTts.kt
+++ b/app/src/main/java/net/kuafuai/andee/tts/HuoshanTts.kt
@@ -34,19 +34,9 @@ import javax.net.ssl.SSLContext
*/
class HuoshanTts(
private val config: VoiceConfig,
- private val listener: Listener,
-) {
- interface Listener {
- fun onOpen(sampleRate: Int)
- fun onAudio(pcm: ByteArray)
- fun onSentenceStart(text: String)
- fun onSentenceEnd(text: String)
- fun onDone()
- fun onError(msg: String)
- fun onClose(reason: CloseReason)
- }
+ private val listener: TtsEngine.Listener,
+) : TtsEngine {
- enum class CloseReason { USER, SERVER, ERROR }
private enum class State { IDLE, CONNECTING, SESSION_OPENING, READY, FINISHING, CLOSED }
private var ws: WebSocketClient? = null
@@ -62,7 +52,7 @@ class HuoshanTts(
private var state: State = State.IDLE
private var sessionId: String = ""
private var connectId: String = ""
- private var failReason: CloseReason = CloseReason.SERVER
+ private var failReason: TtsEngine.CloseReason = TtsEngine.CloseReason.SERVER
private var baseReqParams: JSONObject? = null
@Volatile
@@ -71,7 +61,7 @@ class HuoshanTts(
@Volatile
private var pendingFinishTrigger: (() -> Unit)? = null
- fun open() {
+ override fun open() {
check(state == State.IDLE) { "already open (state=$state)" }
state = State.CONNECTING
@@ -125,7 +115,7 @@ class HuoshanTts(
override fun onError(ex: Exception?) {
Log.e(TAG, "onError", ex)
- failReason = CloseReason.ERROR
+ failReason = TtsEngine.CloseReason.ERROR
listener.onError("ws error: ${ex?.message ?: ex?.javaClass?.simpleName}")
}
}
@@ -150,7 +140,7 @@ class HuoshanTts(
}
/** Push a chunk of text to be spoken. Can be called multiple times. */
- fun push(text: String) {
+ override fun push(text: String) {
if (state != State.READY) {
Log.w(TAG, "push dropped, state=$state")
return
@@ -177,7 +167,7 @@ class HuoshanTts(
* seed-tts-2.0, some speakers don't emit audio until they see FinishSession,
* causing a mutual wait deadlock. Send immediately.
*/
- fun finish() {
+ override fun finish() {
if (state == State.CLOSED || state == State.IDLE) return
if (state != State.READY && state != State.SESSION_OPENING) return
state = State.FINISHING
@@ -185,9 +175,9 @@ class HuoshanTts(
sendEvent(EVT_FINISH_SESSION, "{}".toByteArray(Charsets.UTF_8))
}
- fun cancel() {
+ override fun cancel() {
if (state == State.CLOSED || state == State.IDLE) return
- failReason = CloseReason.USER
+ failReason = TtsEngine.CloseReason.USER
try {
if (state == State.READY || state == State.SESSION_OPENING) {
sendEvent(EVT_CANCEL_SESSION, "{}".toByteArray(Charsets.UTF_8))
@@ -236,7 +226,7 @@ class HuoshanTts(
return
}
if (msg.type == MSG_SERVER_ERROR) {
- failReason = CloseReason.ERROR
+ failReason = TtsEngine.CloseReason.ERROR
listener.onError(
"TTS server error code=${msg.errorCode} " + "msg=${
String(
@@ -258,7 +248,7 @@ class HuoshanTts(
}
EVT_CONNECTION_FAILED -> {
- failReason = CloseReason.ERROR
+ failReason = TtsEngine.CloseReason.ERROR
listener.onError("ConnectionFailed: ${String(msg.payload, Charsets.UTF_8)}")
}
@@ -294,7 +284,7 @@ class HuoshanTts(
}
EVT_SESSION_FAILED -> {
- failReason = CloseReason.ERROR
+ failReason = TtsEngine.CloseReason.ERROR
listener.onError("SessionFailed: ${String(msg.payload, Charsets.UTF_8)}")
}
diff --git a/app/src/main/java/net/kuafuai/andee/tts/TtsController.kt b/app/src/main/java/net/kuafuai/andee/tts/TtsController.kt
index 95f0d99..5fdcc89 100644
--- a/app/src/main/java/net/kuafuai/andee/tts/TtsController.kt
+++ b/app/src/main/java/net/kuafuai/andee/tts/TtsController.kt
@@ -13,9 +13,10 @@ import net.kuafuai.andee.ui.FloatingWindowUi
import java.util.concurrent.Executors
/**
- * One-shot TTS: speak a single string via 火山 bidi TTS, stream PCM to AudioIO.
+ * One-shot TTS: speak a single string through whichever engine the settings
+ * name, stream PCM to AudioIO.
*
- * IDLE ── speak(text) ─→ SPEAKING (WS open, push, finish)
+ * IDLE ── speak(text) ─→ SPEAKING (engine open, push, finish)
* │
* ├── sentence_start → subtitle SPEAKING
* ├── done → IDLE
@@ -30,7 +31,7 @@ class TtsController(
private val window: FloatingWindowUi,
) {
@Volatile
- private var client: HuoshanTts? = null
+ private var client: TtsEngine? = null
// Set by [stopSpeaking] so the in-flight websocket callbacks of an
// interrupted utterance stop touching the UI. Without it, cancelling the
@@ -311,7 +312,7 @@ class TtsController(
ui.post { cueNext = 0 }
// Before the socket opens, not when the first audio lands: a media app
- // takes a moment to actually stop, and the whole 火山 handshake is that
+ // takes a moment to actually stop, and the engine's own start-up is that
// moment. Losing the floor mid-answer (a call arriving) ends the
// utterance — talking underneath a ringtone helps nobody.
//
@@ -324,7 +325,7 @@ class TtsController(
window.setState(FloatingWindowUi.State.IDLE)
}
- val listener = object : HuoshanTts.Listener {
+ val listener = object : TtsEngine.Listener {
override fun onOpen(sampleRate: Int) {
if (cancelled) return
audio.startStreamPlayback(sampleRate)
@@ -367,7 +368,7 @@ class TtsController(
// and it is read on the same thread the audio arrived on.
cueTotalMs = audio.queuedMs()
// SessionFinished means the *server* has stopped sending, not
- // that the user has stopped hearing: 火山 delivers audio far
+ // that the user has stopped hearing: every engine delivers audio far
// faster than real time, so most of the utterance is usually
// still queued at this point. The ball therefore goes IDLE on
// the other side of the drain, not here — otherwise it drops
@@ -395,7 +396,7 @@ class TtsController(
reportUnavailable(msg)
}
- override fun onClose(reason: HuoshanTts.CloseReason) {
+ override fun onClose(reason: TtsEngine.CloseReason) {
// Every planned exit (stopSpeaking, cancel, onDone, onError)
// nulls [client] before the socket actually closes. Reaching
// here with it still set therefore means the socket died on
@@ -412,7 +413,11 @@ class TtsController(
}
}
- val c = HuoshanTts(cfg, listener)
+ // Which service speaks. Same fail-to-火山 rule as the listening side.
+ val c: TtsEngine = when (cfg.voiceProvider) {
+ VoiceConfig.VOICE_GOOGLE -> GoogleTts(cfg, listener)
+ else -> HuoshanTts(cfg, listener)
+ }
client = c
c.open()
}
diff --git a/app/src/main/java/net/kuafuai/andee/tts/TtsEngine.kt b/app/src/main/java/net/kuafuai/andee/tts/TtsEngine.kt
new file mode 100644
index 0000000..7f73410
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/tts/TtsEngine.kt
@@ -0,0 +1,54 @@
+package net.kuafuai.andee.tts
+
+/**
+ * One utterance, whichever backend speaks it.
+ *
+ * The call sequence is 火山's and both backends honour it: [open], then
+ * [push] with the text (once, from the driver's `onOpen`) — for a backend that
+ * synthesises on demand, however many times — then [finish]. Audio comes back
+ * through [Listener.onAudio] as raw 16-bit little-endian PCM.
+ *
+ * **What a backend cannot promise is when the text arrives relative to the
+ * network call.** 火山 accepts text into an open socket, so it can report its
+ * sample rate before it has seen a word; a backend that sends one HTTP request
+ * per utterance only learns the rate from the response. [Listener.onOpen] is
+ * therefore called *before* [push] by construction — see `GoogleTts` for what
+ * that forces it to declare up front.
+ */
+interface TtsEngine {
+
+ /** Why a session ended. Narrated to nobody; used to describe a dead socket. */
+ enum class CloseReason { USER, SERVER, ERROR }
+
+ interface Listener {
+ /**
+ * The session is up and [sampleRate] is what the PCM will be in.
+ *
+ * Called once, before any [onAudio], and — for every backend — before
+ * the driver can push text, because the driver pushes from this
+ * callback.
+ */
+ fun onOpen(sampleRate: Int)
+
+ fun onAudio(pcm: ByteArray)
+
+ fun onSentenceStart(text: String)
+
+ fun onSentenceEnd(text: String)
+
+ /** No more audio is coming. The driver keeps its own drain clock. */
+ fun onDone()
+
+ fun onError(msg: String)
+
+ fun onClose(reason: CloseReason)
+ }
+
+ fun open()
+
+ fun push(text: String)
+
+ fun finish()
+
+ fun cancel()
+}
diff --git a/app/src/main/java/net/kuafuai/andee/ui/SettingsUi.kt b/app/src/main/java/net/kuafuai/andee/ui/SettingsUi.kt
index 91c1293..abd08c2 100644
--- a/app/src/main/java/net/kuafuai/andee/ui/SettingsUi.kt
+++ b/app/src/main/java/net/kuafuai/andee/ui/SettingsUi.kt
@@ -686,27 +686,159 @@ class SettingsUi(
)
note(localBlock, lctx.getString(R.string.settings_brain_local_note))
+ // ---- Which service hears and speaks ----
+ //
+ // One container, so that the whole block can be taken out in one move
+ // rather than view by view — see the visibility flip at the end of it.
+ //
+ // **Not inside `localBlock`**, even though it reads like a 本机
+ // concern: who hears and who thinks are two different questions, and
+ // hanging this off the 本机 tab would hide it from a 云端 device that
+ // legitimately wants to point its own voice service somewhere.
+ //
+ // The two sets of rows are mutually exclusive and are not two flavours
+ // of the same thing: 火山's are endpoints and resource ids, Google's are
+ // a project, a location and two model names.
+ val voiceBlock = block(panel)
+ rowLabel(voiceBlock, lctx.getString(R.string.settings_voice_provider))
+ var applyProvider: ((String, Boolean) -> Unit)? = null
+ val provider = segmented(
+ voiceBlock,
+ "voice_provider",
+ listOf(
+ // 火山 first, because the first option is the fallback when
+ // nothing is saved — and for every device that predates this
+ // row, 火山 is what it has been running. See
+ // [VoiceConfig.voiceProvider].
+ VoiceConfig.VOICE_HUOSHAN to lctx.getString(R.string.settings_voice_provider_huoshan),
+ VoiceConfig.VOICE_GOOGLE to lctx.getString(R.string.settings_voice_provider_google),
+ ),
+ ) { applyProvider?.invoke(it, true) }
+
// 火山: previously in 高级 → 火山. Rows copy that section's labels,
// which still describe exactly these keys.
+ val huoshanBlock = block(voiceBlock)
field(
- localBlock,
+ huoshanBlock,
"api_key",
lctx.getString(R.string.settings_voice_api_key),
hint = lctx.getString(R.string.settings_voice_api_key_hint),
secret = true,
)
- field(localBlock, "asr_endpoint", lctx.getString(R.string.settings_voice_asr_endpoint))
- field(localBlock, "asr_resource_id", lctx.getString(R.string.settings_voice_asr_resource_id))
- field(localBlock, "tts_endpoint", lctx.getString(R.string.settings_voice_tts_endpoint))
- field(localBlock, "tts_resource_id", lctx.getString(R.string.settings_voice_tts_resource_id))
- field(localBlock, "tts_sample_rate", lctx.getString(R.string.settings_voice_tts_sample_rate), numeric = true)
+ field(huoshanBlock, "asr_endpoint", lctx.getString(R.string.settings_voice_asr_endpoint))
+ field(huoshanBlock, "asr_resource_id", lctx.getString(R.string.settings_voice_asr_resource_id))
+ field(huoshanBlock, "tts_endpoint", lctx.getString(R.string.settings_voice_tts_endpoint))
+ field(huoshanBlock, "tts_resource_id", lctx.getString(R.string.settings_voice_tts_resource_id))
+ field(
+ huoshanBlock,
+ "tts_sample_rate",
+ lctx.getString(R.string.settings_voice_tts_sample_rate),
+ numeric = true,
+ )
+
+ // ---- Google: one Vertex Express key, two protocols ----
+ //
+ // Every hint here is the value a blank row resolves to, from the same
+ // constants [VoiceConfig.load] reads — the same rule the LLM rows
+ // follow, for the same reason: a card that advertises a model but
+ // silently calls another is lying where the user goes to decide.
+ // A blank row means "use whatever the build carries", which for a
+ // packaged-for-Google APK is a real key and a real project number. So
+ // the hint has to say that, or the user reads an empty box as "not set
+ // up" while the device is happily talking to Google with a credential
+ // they never typed. The key's *value* is never shown — a settings card
+ // is not a place to print a secret — but the project number is not one,
+ // and printing it is how the row can be checked at a glance.
+ val builtInKey = VoiceConfig.googleApiKeyDefault()
+ val builtInProject = VoiceConfig.googleProjectDefault()
+
+ val googleBlock = block(voiceBlock)
+ field(
+ googleBlock,
+ "google_api_key",
+ lctx.getString(R.string.settings_google_api_key),
+ hint = if (builtInKey.isNotEmpty()) {
+ lctx.getString(R.string.settings_google_key_builtin)
+ } else {
+ lctx.getString(R.string.settings_google_api_key_hint)
+ },
+ secret = true,
+ )
+ field(
+ googleBlock,
+ "google_project",
+ lctx.getString(R.string.settings_google_project),
+ hint = builtInProject.ifEmpty {
+ lctx.getString(R.string.settings_google_project_hint)
+ },
+ )
+ field(
+ googleBlock,
+ "google_location",
+ lctx.getString(R.string.settings_google_location),
+ hint = VoiceConfig.googleLocationDefault(),
+ )
+ field(
+ googleBlock,
+ "google_asr_model",
+ lctx.getString(R.string.settings_google_asr_model),
+ hint = VoiceConfig.googleAsrModelDefault(),
+ )
+ field(
+ googleBlock,
+ "google_tts_model",
+ lctx.getString(R.string.settings_google_tts_model),
+ hint = VoiceConfig.googleTtsModelDefault(),
+ )
+ field(
+ googleBlock,
+ "google_base_url",
+ lctx.getString(R.string.settings_google_base_url),
+ hint = VoiceConfig.googleBaseUrlDefault(),
+ )
+ field(
+ googleBlock,
+ "google_language",
+ lctx.getString(R.string.settings_google_language),
+ hint = lctx.getString(R.string.settings_google_language_hint),
+ )
+ note(googleBlock, lctx.getString(R.string.settings_voice_google_note))
+ applyProvider = { p, animate ->
+ Glass.setVisible(huoshanBlock, p == VoiceConfig.VOICE_HUOSHAN, animate)
+ Glass.setVisible(googleBlock, p == VoiceConfig.VOICE_GOOGLE, animate)
+ }
+ applyProvider(provider, false)
+
+ // **The voice block is not drawn at all under the hosted backend**, and
+ // not merely disabled. There, [VoiceConfig.voiceProvider] pins 火山 and
+ // [VoiceConfig.load] takes every endpoint and credential from the build
+ // — so every control in this block is read by nothing. A row the user
+ // can edit that changes no behaviour is the small lie this card exists
+ // to avoid, and the honest surface for "what is actually in use" is the
+ // self-check page, which grades the real provider rather than a saved
+ // string.
+ //
+ // **Its flip belongs to the mode picker, not to this build pass.** It
+ // used to be a lone `setVisible` evaluated once here, against the
+ // *saved* mode, so picking 本机 out of CodeFlying left the block hidden
+ // until 保存 rebuilt the card — a control that only appears if you
+ // press the button standing next to it. Same condition; the difference
+ // is that it is asked of the value the user just picked, through
+ // [VoiceConfig.resolveBrainMode], so a build that does not enable
+ // CodeFlying still shows the block its runtime is actually reading.
+ //
// One block visible at a time. Simultaneous flips so the card's total
// height barely moves — see the old comment, still true.
applyMode = { m, animate ->
Glass.setVisible(hubBlock, m == VoiceConfig.BRAIN_HUB, animate)
Glass.setVisible(cfBlock, m == VoiceConfig.BRAIN_CODEFLYING, animate)
Glass.setVisible(localBlock, m == VoiceConfig.BRAIN_LOCAL, animate)
+ Glass.setVisible(
+ voiceBlock,
+ VoiceConfig.resolveBrainMode(m) != VoiceConfig.BRAIN_CODEFLYING,
+ animate,
+ )
}
applyMode(mode, false)
applyThinking = { th, animate ->
diff --git a/app/src/main/res/values-en/strings.xml b/app/src/main/res/values-en/strings.xml
index 97050b1..383f5c3 100644
--- a/app/src/main/res/values-en/strings.xml
+++ b/app/src/main/res/values-en/strings.xml
@@ -120,6 +120,21 @@
<string name="settings_voice_tts_endpoint">TTS endpoint</string>
<string name="settings_voice_tts_resource_id">TTS resource ID</string>
<string name="settings_voice_tts_sample_rate">TTS sample rate</string>
+ <string name="settings_voice_provider">Voice provider</string>
+ <string name="settings_voice_provider_huoshan">Volcengine</string>
+ <string name="settings_voice_provider_google">Google</string>
+ <string name="settings_google_api_key">Google API key</string>
+ <string name="settings_google_api_key_hint">Vertex Express key — hearing and speaking both use it</string>
+ <string name="settings_google_key_builtin">Built in at package time — leave blank to use it</string>
+ <string name="settings_google_project">Project number</string>
+ <string name="settings_google_project_hint">The GCP project *number* (not its id); recognition needs it</string>
+ <string name="settings_google_location">Location</string>
+ <string name="settings_google_asr_model">Recognition model</string>
+ <string name="settings_google_tts_model">Synthesis model</string>
+ <string name="settings_google_base_url">Endpoint</string>
+ <string name="settings_google_language">Language</string>
+ <string name="settings_google_language_hint">blank = detect automatically</string>
+ <string name="settings_voice_google_note">Recognition runs on Gemini Live (so it needs the project number and location); synthesis is plain HTTP. The key stays on this device.</string>
<string name="settings_section_diagnostics">Diagnostics</string>
<string name="settings_section_advanced">Advanced</string>
<string name="settings_max_steps">Max steps per task</string>
@@ -482,8 +497,8 @@
<string name="check_detail_secure_spare">Not granted, and typing works anyway — it goes through the accessibility input method. This is only needed by the spare channel above.</string>
<string name="check_detail_mic_on">Allowed.</string>
<string name="check_detail_mic_off">Not allowed. Tapping the ball does nothing at all — the failure that least resembles a permission problem.</string>
- <string name="check_detail_voice_key_ok">Set. Recognition and synthesis use it to reach Volcengine.</string>
- <string name="check_detail_voice_key_missing">Not set. The source ships without a key — open a Volcengine Speech service of your own and put the key in ⚙ → Backend → On-device → “Volcengine API key”. Without it, hearing and speaking cannot connect, and what you get back is a connection timeout that looks like a network fault. Everything else works.</string>
+ <string name="check_detail_voice_key_ok">Set. Recognition and synthesis use it to reach the provider.</string>
+ <string name="check_detail_voice_key_missing">Not set. The source ships without a key — open a speech service of your own and put its key in ⚙ → Backend → On-device. Without it, hearing and speaking cannot connect, and what you get back is a connection timeout that looks like a network fault. Everything else works.</string>
<string name="check_detail_voice_host_ok">%1$s is reachable.</string>
<string name="check_detail_voice_host_bad">Could not reach %1$s: %2$s. Both recognition and synthesis will time out.</string>
<string name="check_detail_wake_on">Taught, and it wakes when you say it.</string>
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index 260956d..9ee61cf 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -97,6 +97,21 @@
<string name="settings_voice_tts_endpoint">合成地址</string>
<string name="settings_voice_tts_resource_id">合成资源 ID</string>
<string name="settings_voice_tts_sample_rate">合成采样率</string>
+ <string name="settings_voice_provider">语音服务商</string>
+ <string name="settings_voice_provider_huoshan">火山</string>
+ <string name="settings_voice_provider_google">谷歌</string>
+ <string name="settings_google_api_key">Google API Key</string>
+ <string name="settings_google_api_key_hint">Vertex Express Key,听和说都用它</string>
+ <string name="settings_google_key_builtin">已内置(编译时写入),留空即用它</string>
+ <string name="settings_google_project">项目号</string>
+ <string name="settings_google_project_hint">GCP 的项目编号(不是项目名),识别要用</string>
+ <string name="settings_google_location">区域</string>
+ <string name="settings_google_asr_model">识别模型</string>
+ <string name="settings_google_tts_model">合成模型</string>
+ <string name="settings_google_base_url">接口地址</string>
+ <string name="settings_google_language">语言</string>
+ <string name="settings_google_language_hint">留空 = 自动判断</string>
+ <string name="settings_voice_google_note">识别走 Gemini Live(需要项目号和区域),合成走 HTTP。密钥只存在这台设备上。</string>
<string name="settings_section_diagnostics">诊断</string>
<string name="settings_section_advanced">高级</string>
<string name="settings_max_steps">单次任务最多步数</string>
@@ -564,8 +579,8 @@
<string name="check_detail_secure_spare">没授权,不过不影响打字 —— 现在走的是无障碍输入法。它只给上面那条备用通道用,没有也行。</string>
<string name="check_detail_mic_on">已允许。</string>
<string name="check_detail_mic_off">没允许。点球不会有任何反应 —— 这是最不像权限问题的一条。</string>
- <string name="check_detail_voice_key_ok">已填。识别和合成用它连火山。</string>
- <string name="check_detail_voice_key_missing">还没填。源码里不带密钥,得你自己去火山引擎开一个语音服务,把 Key 填进 ⚙ →「后端」→「本机」→「火山 API Key」。没有它,听和说都连不上 —— 报出来的却是连接超时,看着像网络坏了。其余功能照常。</string>
+ <string name="check_detail_voice_key_ok">已填。识别和合成用它连服务商。</string>
+ <string name="check_detail_voice_key_missing">还没填。源码里不带密钥,得你自己去开一个语音服务,把 Key 填进 ⚙ →「后端」→「本机」。没有它,听和说都连不上 —— 报出来的却是连接超时,看着像网络坏了。其余功能照常。</string>
<string name="check_detail_voice_host_ok">%1$s 可达。</string>
<string name="check_detail_voice_host_bad">连不上 %1$s:%2$s。识别和合成都会超时。</string>
<string name="check_detail_wake_on">已录入,喊一声就醒。</string>
diff --git a/build-and-install.sh b/build-and-install.sh
new file mode 100755
index 0000000..7b0dafd
--- /dev/null
+++ b/build-and-install.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+#
+# 打包 andee 并装到平板。
+#
+# 一把过:查设备 → 验后端连不连得通 → 打包 → 装对应架构的包。
+#
+# ./build-and-install.sh # 用 local.properties 里的配置
+# ./build-and-install.sh 192.168.1.5 # 顺手把后端地址改成这个再打
+#
+# 后端地址写在 local.properties,所以这个脚本不猜任何东西 —— 它只负责把那份
+# 配置打进去、装上去,然后把「平板能不能连到后端」这件事先验掉。装完才发现连不上
+# 是最浪费时间的,所以那一步放在最前面。
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+ADB="${ADB:-$HOME/Library/Android/sdk/platform-tools/adb}"
+JAVA_HOME="${JAVA_HOME:-$HOME/Library/Java/JavaVirtualMachines/corretto-17.0.13/Contents/Home}"
+export JAVA_HOME
+
+PROPS=local.properties
+
+# ---- 0. 可选:改后端地址 --------------------------------------------------
+if [ $# -ge 1 ]; then
+ domain="$1"
+ # 回环不带端口的话补上 8080。
+ #
+ # **端口必须写进域名里**,因为 app 是把 DOMAIN 直接拼进 URL 的
+ # (VoiceConfig.codeFlyingBaseUrl = "http://" + DOMAIN),不带端口就等于 80。
+ # 而 80 上什么都没有,报出来的是一句「无法连接」,看不出是端口的事。踩过。
+ case "$domain" in
+ *:*) ;;
+ 127.0.0.1|localhost) domain="${domain}:8080" ;;
+ esac
+ sed -i '' -E "s|^CODEFLYING_DOMAIN=.*|CODEFLYING_DOMAIN=${domain}|" "$PROPS"
+ sed -i '' -E "s|^CODEFLYING_SSL_ENABLED=.*|CODEFLYING_SSL_ENABLED=false|" "$PROPS"
+ echo "后端地址已改为 ${domain}(SSL 关闭)"
+fi
+
+host=$(grep -E '^CODEFLYING_DOMAIN=' "$PROPS" | cut -d= -f2)
+ssl=$(grep -E '^CODEFLYING_SSL_ENABLED=' "$PROPS" | cut -d= -f2)
+hostonly=${host%%:*}
+key=$(grep -E '^CODEFLYING_KEY=' "$PROPS" | cut -d= -f2)
+
+# 端口和 scheme 只在探测连通性时用得上。
+#
+# **不要给真实域名默认 8080。** 这里原来写死了 ${port:-8080} —— 那是为
+# `adb reverse`(回环 + 8080)准备的,用到真域名上就把 http://host 探成了
+# host:8080,永远不通,而报出来的是一句「平板连不到这台机器」,指向网络。
+# 真域名不带端口就是走默认端口(http 80 / https 443)。
+scheme=$([ "$ssl" = "true" ] && echo https || echo http)
+probe="${scheme}://${host}"
+if [ "$hostonly" = "127.0.0.1" ] || [ "$hostonly" = "localhost" ]; then
+ revport="${host##*:}"
+ [ "$revport" = "$hostonly" ] && revport=8080
+fi
+
+echo "════════════════════════════════════════"
+echo " 后端 ${probe}"
+echo " 令牌 ${key:0:12}…"
+echo "════════════════════════════════════════"
+echo
+
+# ---- 1. 设备 --------------------------------------------------------------
+if ! "$ADB" get-state >/dev/null 2>&1; then
+ echo "✗ 没有设备。插上 USB,或者 adb connect <平板IP>:5555"
+ exit 1
+fi
+model=$("$ADB" shell getprop ro.product.model 2>/dev/null | tr -d '\r')
+abi=$("$ADB" shell getprop ro.product.cpu.abi 2>/dev/null | tr -d '\r')
+echo "设备 ${model} (${abi})"
+
+# ---- 2. 后端地址是回环的话,先建 USB 反向转发 ------------------------------
+# 平板和电脑不在同一个网的时候用这个:adbd 把平板的 127.0.0.1:PORT 转回这台机器,
+# 整条路走 USB,不经过任何网络。所以两边在什么网段都无所谓。
+#
+# 无线方案在这里没用 —— reverse 要的是一条稳定的 adb 通道,而无线 adb 本身
+# 就要求同网,那就回到原问题了。
+if [ -n "${revport:-}" ]; then
+ echo -n "转发 USB ${revport} → 本机 ${revport} … "
+ if "$ADB" reverse "tcp:${revport}" "tcp:${revport}" >/dev/null 2>&1; then
+ echo "通"
+ else
+ echo "失败 —— USB 没插或者 adb 掉了"
+ exit 1
+ fi
+fi
+
+# ---- 3. 平板能不能连到后端 ------------------------------------------------
+# 先验这个。打包两分钟、装包一分钟,装完才发现网不通是最亏的。
+echo -n "连通 ${probe} … "
+if "$ADB" shell "curl -s -o /dev/null -w '%{http_code}' -m 8 '${probe}/'" 2>/dev/null | grep -qE '^[0-9]'; then
+ echo "通"
+else
+ echo "不通"
+ echo
+ echo " 平板连不到这台机器。常见原因:"
+ echo " · 不在同一个 WiFi(平板插着 USB 不代表网络也在一起)"
+ echo " · macOS 防火墙拦了 java"
+ echo " · baas 没在跑"
+ echo
+ echo " 平板自己的地址:"
+ "$ADB" shell ip -4 addr show 2>/dev/null | grep 'inet ' | grep -v 127.0.0.1 | sed 's/^/ /' || true
+ echo
+ read -rp " 还是继续打包?[y/N] " ans
+ [ "$ans" = "y" ] || exit 1
+fi
+
+# ---- 4. 打包 --------------------------------------------------------------
+echo -n "打包 "
+./gradlew assembleDebug -q 2>&1 | grep -E "error:|FAILED" && { echo "✗ 打包失败"; exit 1; }
+echo "完成"
+
+apk="app/build/outputs/apk/debug/app-${abi}-debug.apk"
+[ -f "$apk" ] || apk=$(ls app/build/outputs/apk/debug/app-*-debug.apk 2>/dev/null | head -1)
+echo "产物 $apk"
+
+grep -E 'CODEFLYING_(DOMAIN|SSL)' app/build/generated/source/buildConfig/debug/net/kuafuai/andee/BuildConfig.java \
+ | sed 's/^/ /'
+
+# ---- 5. 安装 --------------------------------------------------------------
+# 从 IDEA 或别的机器装过的包签名可能对不上,装不上就卸了重装。
+# 卸载会清掉聊天记录和小本本 —— 测试环境没关系,所以这里不拦。
+echo -n "安装 "
+out=$("$ADB" install -r "$apk" 2>&1 || true)
+
+if echo "$out" | grep -q "Success"; then
+ echo "完成(覆盖安装,数据保留)"
+elif echo "$out" | grep -qE "no devices|device offline|device unauthorized"; then
+ # 设备掉线。这一步是独立的失败,绝不能滑到下面的「卸载重装」去 ——
+ # 那会白白清掉平板上的聊天记录和小本本。踩过。
+ echo "✗ 设备掉线了(USB 松了?锁屏了?)"
+ echo " $out"
+ exit 1
+elif echo "$out" | grep -q "signatures do not match"; then
+ echo "签名对不上 —— 卸了重装(聊天记录和小本本会清)"
+ "$ADB" uninstall net.kuafuai.andee >/dev/null 2>&1 || true
+ "$ADB" install "$apk" 2>&1 | tail -1
+else
+ echo "✗ 装不上"
+ echo " $out"
+ exit 1
+fi
+
+echo
+echo "装好了。语音那条链路是走 ${host} 的 —— 记得那边的 baas 在跑、且是最新代码。"