Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。
feat(card): open the mic after an ask_user question and take any spoken answer
booboosui03c9c247a2e3
diff --git a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.ktindex dc295c4..46e54b0 100644--- a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt+++ b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt@@ -444,7 +444,7 @@ class ScreenBodyService : AccessibilityService() {onPageClosed = { title, seconds -> notifyPageClosed(title, seconds) },// The ask/alert cards speak their text; the mute rules live here,// not in the dispatcher. See [speakCardText].- sayAloud = { text -> speakCardText(text) },+ sayAloud = { text, listen -> speakCardText(text, listen) },)// Extension coming or going invalidates the hub's cached tool list; the// hub has no incremental protocol, so we bounce the outbound client and@@ -732,7 +732,9 @@ class ScreenBodyService : AccessibilityService() {// rather than start a new turn — the user spoke to the question on// screen, not to the brain. Match is fuzzy (case-insensitive, prefix),// so "是" / "yes" / "ok" all hit a button labelled "是" or "Yes" or "OK".- if (how == "voice" && net.kuafuai.andee.ui.CardUi.tryAnswerByVoice(said)) {+ if (how == "voice" &&+ net.kuafuai.andee.ui.CardUi.tryAnswerByVoice(said) != net.kuafuai.andee.ui.CardUi.VoiceOutcome.NOT_HANDLED+ ) {android.util.Log.i("Body", "$how answered card: $said")return}@@ -2157,20 +2159,22 @@ class ScreenBodyService : AccessibilityService() {* refusal. Its caller is usually a person recording a demo, and "nothing* happened and nothing said why" is the worst answer to give them.*/- private fun speakCardText(text: String): Boolean {+ private fun speakCardText(text: String, listen: Boolean = false): Boolean {if (!acceptBrainOutput) return falseif (meeting.isActive() || net.kuafuai.andee.device.CallState.isActive) return false- // A card is its own answer channel — the ask's buttons, the alert's- // 知道了. The follow-up mic must not open over it: a spoken answer- // would arrive as a *new turn* queued behind the very tool call the- // card is blocking, and a mic held open while the user decides- // overhears the room. The user who does want to talk has the wake- // word and the ball.+ // [listen] is a `ui.ask`: the brain is blocked on this very card and+ // the user answers it by voice. [CardUi.tryAnswerByVoice] is what makes+ // that safe, because a sentence spoken while the card is up is+ // delivered to the card and never starts a turn of its own, which would+ // queue behind the tool call the card is blocking. That is why the+ // follow-up mic may open here. It listens once: [AsrController]'s+ // silence watchdog closes it, and an unanswered question still has the+ // buttons, the ball and the wake word.//- // True for `tts.speak` as well, and for a different version of the same- // reason: a rehearsal line is not a question the assistant asked, so- // the mic must not come up hunting for an answer to it.- expectFollowUp = false+ // Everything else stays shut: an `ui.alert` asks nothing, and a+ // `tts.speak` rehearsal line is not a question the assistant asked, so+ // the mic must not come up hunting for an answer to either.+ expectFollowUp = listenval script = SpeechMood.parse(stripMarkdown(text))tts.speak(script.text, script.cues)return truediff --git a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.ktindex f5197db..4726610 100644--- a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt+++ b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt@@ -113,7 +113,7 @@ class CommandDispatcher(* The card is for a user who may be looking anywhere but the screen;* a question nobody hears just waits in silence for its timeout.*/- private val sayAloud: (String) -> Boolean = { false },+ private val sayAloud: (text: String, listen: Boolean) -> Boolean = { _, _ -> false },) {private val ui = Handler(Looper.getMainLooper())@@ -527,7 +527,7 @@ class CommandDispatcher(// this says so instead of reporting a success nobody can hear."tts.speak" -> {val text = requireNotNull(params?.optString("text")?.ifEmpty { null }) { "tts.speak requires text" }- val spoken = sayAloud(text)+ val spoken = sayAloud(text, false)JSONObject().put("spoken", spoken).put("chars", text.length)@@ -560,20 +560,37 @@ class CommandDispatcher(// worker — bridge through a latch.val latch = java.util.concurrent.CountDownLatch(1)var answer: net.kuafuai.andee.ui.CardUi.Result? = null- net.kuafuai.andee.ui.CardUi.ask(question, buttons, timeoutMs) { r ->+ // captureSpeech: this call is what the brain is blocked on, so+ // words that name no button still answer it, and come back in+ // the result below for the model to read against the options.+ net.kuafuai.andee.ui.CardUi.ask(question, buttons, timeoutMs, captureSpeech = true) { r ->answer = r; latch.countDown()}// The card is up in the user's hands, but they may be looking// anywhere else — the ball is a voice-first device, so the// question is spoken as well as shown. The service applies the- // same mute rules it applies to the final answer.- sayAloud(question)+ // same mute rules it applies to the final answer. `listen`:+ // when the question has been read out the mic opens by itself,+ // so the user can just answer.+ sayAloud(question, true)latch.await(timeoutMs + 3000, java.util.concurrent.TimeUnit.MILLISECONDS)val r = answer ?: net.kuafuai.andee.ui.CardUi.Result(null, "no_answer")org.json.JSONObject().put("button", r.button).put("how", r.how)- .put("answered", r.button != null)+ .put("answered", r.button != null || r.heard != null)+ .apply {+ if (r.heard != null) {+ put("heard", r.heard)+ if (r.button == null) {+ put(+ "note",+ "The user answered by voice and their words match none of the " ++ "buttons. Read \"heard\" as their answer to your question.",+ )+ }+ }+ }}"ui.alert" -> {val text = requireNotNull(params?.optString("text")?.ifEmpty { null }) { "ui.alert requires text" }@@ -585,7 +602,7 @@ class CommandDispatcher(}// Same voice-first reasoning as ui.ask: a notice nobody hears// is a notice that was never delivered.- sayAloud(text)+ sayAloud(text, false)latch.await(timeoutMs + 3000, java.util.concurrent.TimeUnit.MILLISECONDS)val r = answer ?: net.kuafuai.andee.ui.CardUi.Result(null, "no_answer")org.json.JSONObject()diff --git a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.ktindex 1b7d87b..f7bf36d 100644--- a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt+++ b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt@@ -780,9 +780,17 @@ object ToolSchemas {method = "ui.ask",description = """Ask the user a question on a small floating card with tappable- buttons — blocks until they answer (or timeout). Returns which- button they tapped. The question is also read aloud (TTS), so- write it as one speakable sentence, no markdown or lists.+ buttons — blocks until they answer (or timeout). The question is+ also read aloud (TTS) and the microphone opens afterwards, so the+ user may simply answer out loud. Write it as one speakable+ sentence, no markdown or lists.++ The result says how they answered. `button` is the button they+ tapped or named. If they said something that matches no button,+ `button` is null and `heard` holds their words: take those as+ their answer and act on what they mean, which may be a choice, a+ different option, or "neither". `answered` false means no reply+ (timeout or dismissed).USE WHEN the next step genuinely needs the user's choice: which ofseveral options, yes/no on something irreversible, "要继续吗".diff --git a/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt b/app/src/main/java/net/kuafuai/andee/ui/CardUi.ktindex 394908d..e4719fe 100644--- a/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt+++ b/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt@@ -92,9 +92,12 @@ object CardUi {}class Result(- /** Which button the user tapped, or null for timeout/superseded/dismissed-elsewhere. */+ /** Which button the user tapped or named, or null for timeout/superseded/dismissed-elsewhere. */val button: String?,- val how: String, // "button" | "timeout" | "superseded" | "tap_outside"+ /** "button" | "voice" | "voice_other" | "timeout" | "superseded" | "tap_outside" */+ val how: String,+ /** The user's own words, when they answered by voice. */+ val heard: String? = null,)/**@@ -108,9 +111,15 @@ object CardUi {question: String,buttons: List<String>,timeoutMs: Long = AUTO_DISMISS_MS,+ // True only for a question the brain is blocked on (`ui.ask`). Speech+ // that names no button then closes the card and travels back in the tool+ // result for the model to judge. Every other card (a notification's+ // 看看/忽略, the self-check nudge) has nobody waiting for words, so+ // unmatched speech must stay an ordinary turn and leave the card alone.+ captureSpeech: Boolean = false,onDone: (Result) -> Unit,) {- show(question, buttons, cancelable = false, timeoutMs, onDone)+ show(question, buttons, cancelable = false, timeoutMs, captureSpeech, onDone)}/**@@ -122,7 +131,7 @@ object CardUi {// same condition one call later — so returning here changes nothing// observable, it just keeps the label lookup off a null.val ctx = appContext ?: return- show(text, listOf(AppLocale.str(ctx, R.string.card_ack)), cancelable = true, timeoutMs, onDone)+ show(text, listOf(AppLocale.str(ctx, R.string.card_ack)), cancelable = true, timeoutMs, false, onDone)}/**@@ -151,48 +160,76 @@ object CardUi {main.post { current?.close("superseded") }}+ /** What [tryAnswerByVoice] did with a spoken sentence. */+ enum class VoiceOutcome {+ /** No card was waiting on speech; the sentence is an ordinary turn. */+ NOT_HANDLED,++ /** It named a button; the card closed as if that button was tapped. */+ BUTTON,++ /** A `ui.ask` card was waiting and the words named no button; the card+ * closed and the words travel back in the tool result for the model. */+ FORWARDED,+ }+/**- * Try to answer the current card by matching [spokenText] against its- * button labels. Returns true if a match was found and the card was closed- * (the caller should not start a new turn); false if no card is up or no- * button matched (the caller may proceed with a normal turn).+ * Answer the card that is up with something the user said.*- * Called from the ASR path when the user speaks while a card is visible.- * The match is case-insensitive and normalised: "是" / "yes" / "确定" all- * hit a button labelled "是" or "Yes" or "确定". Partial matches (spoken- * text is a prefix or the button is a prefix of the spoken text) also- * count, so "好" matches "好的".+ * A sentence that is (nearly) just a button label presses it. A card+ * opened with `captureSpeech` (a `ui.ask`, the brain is blocked on it)+ * treats any other sentence as the answer too: the card closes and the+ * words go back in the tool result, so the model judges them against the+ * options it offered. That is why this never starts a turn of its own: a+ * new turn would queue behind the very tool call this card is blocking.*- * Must be called from any thread; callback happens on the main thread.+ * Matching is deliberately strict. "否则帮我查一下" starts with 否 and is not+ * a no, so a label only counts when it is most of what was said.+ *+ * Blocks the calling thread until the main thread has decided, with no+ * timeout: the caller is on an ASR worker, and giving up early would let+ * the same sentence start a second turn after it already closed the card.*/- fun tryAnswerByVoice(spokenText: String): Boolean {- val text = spokenText.trim().lowercase()- if (text.isEmpty()) return false+ fun tryAnswerByVoice(spokenText: String): VoiceOutcome {+ val said = spokenText.trim()+ if (said.isEmpty()) return VoiceOutcome.NOT_HANDLED+ if (Looper.myLooper() == Looper.getMainLooper()) return decideVoice(said)val latch = java.util.concurrent.CountDownLatch(1)- val matched = java.util.concurrent.atomic.AtomicBoolean(false)+ var outcome = VoiceOutcome.NOT_HANDLEDmain.post {try {- val handle = current ?: return@post- val buttons = handle.buttons- val hit = buttons.firstOrNull { btn ->- val label = btn.trim().lowercase()- label == text || label.startsWith(text) || text.startsWith(label)- }- if (hit != null) {- matched.set(true)- handle.close("voice", hit)- }+ outcome = decideVoice(said)} finally {latch.countDown()}}- // Block until the main thread has decided. The caller is on the ASR- // callback thread and must know whether to bail or proceed before- // returning. Timeout guards against the main thread being dead.- latch.await(200, java.util.concurrent.TimeUnit.MILLISECONDS)- return matched.get()+ latch.await()+ return outcome}+ private fun decideVoice(said: String): VoiceOutcome {+ val handle = current ?: return VoiceOutcome.NOT_HANDLED+ if (handle.finished) return VoiceOutcome.NOT_HANDLED+ val spoken = normalizeForMatch(said)+ val hit = handle.buttons.firstOrNull { label ->+ val l = normalizeForMatch(label)+ l.isNotEmpty() && (spoken == l || (spoken.contains(l) && l.length * 2 >= spoken.length))+ }+ if (hit != null) {+ handle.close("voice", hit, heard = said)+ return VoiceOutcome.BUTTON+ }+ if (handle.captureSpeech) {+ handle.close("voice_other", null, heard = said)+ return VoiceOutcome.FORWARDED+ }+ return VoiceOutcome.NOT_HANDLED+ }++ /** Lower-cased, with spaces and punctuation dropped, so "好的。" equals "好的". */+ private fun normalizeForMatch(s: String): String =+ s.lowercase().filter { it.isLetterOrDigit() }+// ------------------------------------------------------------------private class CardHandle(@@ -201,11 +238,13 @@ object CardUi {val host: FloatingWindowUi?,/** Button labels, in the order they were passed to show(). */val buttons: List<String>,+ /** Whether speech that names no button should still answer this card. */+ val captureSpeech: Boolean = false,@Volatile var finished: Boolean = false,var onDone: ((Result) -> Unit)? = null,var timeout: Runnable? = null,) {- fun close(how: String, button: String? = null) {+ fun close(how: String, button: String? = null, heard: String? = null) {if (finished) returnfinished = truetimeout?.let { main.removeCallbacks(it) }@@ -218,7 +257,7 @@ object CardUi {// The ball can stop attending — the question is answered, timed// out, or was replaced by a newer one (which re-arms it below).ballWindow()?.setHolding(false)- onDone?.invoke(Result(button, how))+ onDone?.invoke(Result(button, how, heard))}}@@ -243,6 +282,7 @@ object CardUi {buttons: List<String>,cancelable: Boolean,timeoutMs: Long,+ captureSpeech: Boolean,onDone: (Result) -> Unit,) {val windowManager = wm ?: return@@ -393,7 +433,7 @@ object CardUi {cardReal.tailX = -1// ---- handle + callbacks ------------------------------------- val handle = CardHandle(cardReal, host = null, buttons = buttons)+ val handle = CardHandle(cardReal, host = null, buttons = buttons, captureSpeech = captureSpeech)handle.onDone = { r -> main.post { onDone(r) } }wireButtons(handle, btnViews, btnLabels, cancelable, cardReal, title)val timeoutRunnable = Runnable { handle.close("timeout") }@@ -431,7 +471,7 @@ object CardUi {cardReal.tailX = cardReal.measuredWidth / 2// ---- handle + callbacks ----------------------------------------- val handle = CardHandle(cardReal, host, buttons = buttons)+ val handle = CardHandle(cardReal, host, buttons = buttons, captureSpeech = captureSpeech)handle.onDone = { r -> main.post { onDone(r) } }wireButtons(handle, btnViews, btnLabels, cancelable, cardReal, title)val timeoutRunnable = Runnable { handle.close("timeout") }