Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(card): open the mic after an ask_user question and take any spoken answer

booboosui03c9c247a2e3
diff --git a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
index dc295c4..46e54b0 100644
--- a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
+++ b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
@@ -444,7 +444,7 @@ class ScreenBodyService : AccessibilityService() {
onPageClosed = { title, seconds -> notifyPageClosed(title, seconds) },
// The ask/alert cards speak their text; the mute rules live here,
// not in the dispatcher. See [speakCardText].
- sayAloud = { text -> speakCardText(text) },
+ sayAloud = { text, listen -> speakCardText(text, listen) },
)
// Extension coming or going invalidates the hub's cached tool list; the
// hub has no incremental protocol, so we bounce the outbound client and
@@ -732,7 +732,9 @@ class ScreenBodyService : AccessibilityService() {
// rather than start a new turn — the user spoke to the question on
// screen, not to the brain. Match is fuzzy (case-insensitive, prefix),
// so "是" / "yes" / "ok" all hit a button labelled "是" or "Yes" or "OK".
- if (how == "voice" && net.kuafuai.andee.ui.CardUi.tryAnswerByVoice(said)) {
+ if (how == "voice" &&
+ net.kuafuai.andee.ui.CardUi.tryAnswerByVoice(said) != net.kuafuai.andee.ui.CardUi.VoiceOutcome.NOT_HANDLED
+ ) {
android.util.Log.i("Body", "$how answered card: $said")
return
}
@@ -2157,20 +2159,22 @@ class ScreenBodyService : AccessibilityService() {
* refusal. Its caller is usually a person recording a demo, and "nothing
* happened and nothing said why" is the worst answer to give them.
*/
- private fun speakCardText(text: String): Boolean {
+ private fun speakCardText(text: String, listen: Boolean = false): Boolean {
if (!acceptBrainOutput) return false
if (meeting.isActive() || net.kuafuai.andee.device.CallState.isActive) return false
- // A card is its own answer channel — the ask's buttons, the alert's
- // 知道了. The follow-up mic must not open over it: a spoken answer
- // would arrive as a *new turn* queued behind the very tool call the
- // card is blocking, and a mic held open while the user decides
- // overhears the room. The user who does want to talk has the wake
- // word and the ball.
+ // [listen] is a `ui.ask`: the brain is blocked on this very card and
+ // the user answers it by voice. [CardUi.tryAnswerByVoice] is what makes
+ // that safe, because a sentence spoken while the card is up is
+ // delivered to the card and never starts a turn of its own, which would
+ // queue behind the tool call the card is blocking. That is why the
+ // follow-up mic may open here. It listens once: [AsrController]'s
+ // silence watchdog closes it, and an unanswered question still has the
+ // buttons, the ball and the wake word.
//
- // True for `tts.speak` as well, and for a different version of the same
- // reason: a rehearsal line is not a question the assistant asked, so
- // the mic must not come up hunting for an answer to it.
- expectFollowUp = false
+ // Everything else stays shut: an `ui.alert` asks nothing, and a
+ // `tts.speak` rehearsal line is not a question the assistant asked, so
+ // the mic must not come up hunting for an answer to either.
+ expectFollowUp = listen
val script = SpeechMood.parse(stripMarkdown(text))
tts.speak(script.text, script.cues)
return true
diff --git a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
index f5197db..4726610 100644
--- a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
+++ b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
@@ -113,7 +113,7 @@ class CommandDispatcher(
* The card is for a user who may be looking anywhere but the screen;
* a question nobody hears just waits in silence for its timeout.
*/
- private val sayAloud: (String) -> Boolean = { false },
+ private val sayAloud: (text: String, listen: Boolean) -> Boolean = { _, _ -> false },
) {
private val ui = Handler(Looper.getMainLooper())
@@ -527,7 +527,7 @@ class CommandDispatcher(
// this says so instead of reporting a success nobody can hear.
"tts.speak" -> {
val text = requireNotNull(params?.optString("text")?.ifEmpty { null }) { "tts.speak requires text" }
- val spoken = sayAloud(text)
+ val spoken = sayAloud(text, false)
JSONObject()
.put("spoken", spoken)
.put("chars", text.length)
@@ -560,20 +560,37 @@ class CommandDispatcher(
// worker — bridge through a latch.
val latch = java.util.concurrent.CountDownLatch(1)
var answer: net.kuafuai.andee.ui.CardUi.Result? = null
- net.kuafuai.andee.ui.CardUi.ask(question, buttons, timeoutMs) { r ->
+ // captureSpeech: this call is what the brain is blocked on, so
+ // words that name no button still answer it, and come back in
+ // the result below for the model to read against the options.
+ net.kuafuai.andee.ui.CardUi.ask(question, buttons, timeoutMs, captureSpeech = true) { r ->
answer = r; latch.countDown()
}
// The card is up in the user's hands, but they may be looking
// anywhere else — the ball is a voice-first device, so the
// question is spoken as well as shown. The service applies the
- // same mute rules it applies to the final answer.
- sayAloud(question)
+ // same mute rules it applies to the final answer. `listen`:
+ // when the question has been read out the mic opens by itself,
+ // so the user can just answer.
+ sayAloud(question, true)
latch.await(timeoutMs + 3000, java.util.concurrent.TimeUnit.MILLISECONDS)
val r = answer ?: net.kuafuai.andee.ui.CardUi.Result(null, "no_answer")
org.json.JSONObject()
.put("button", r.button)
.put("how", r.how)
- .put("answered", r.button != null)
+ .put("answered", r.button != null || r.heard != null)
+ .apply {
+ if (r.heard != null) {
+ put("heard", r.heard)
+ if (r.button == null) {
+ put(
+ "note",
+ "The user answered by voice and their words match none of the " +
+ "buttons. Read \"heard\" as their answer to your question.",
+ )
+ }
+ }
+ }
}
"ui.alert" -> {
val text = requireNotNull(params?.optString("text")?.ifEmpty { null }) { "ui.alert requires text" }
@@ -585,7 +602,7 @@ class CommandDispatcher(
}
// Same voice-first reasoning as ui.ask: a notice nobody hears
// is a notice that was never delivered.
- sayAloud(text)
+ sayAloud(text, false)
latch.await(timeoutMs + 3000, java.util.concurrent.TimeUnit.MILLISECONDS)
val r = answer ?: net.kuafuai.andee.ui.CardUi.Result(null, "no_answer")
org.json.JSONObject()
diff --git a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt
index 1b7d87b..f7bf36d 100644
--- a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt
+++ b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt
@@ -780,9 +780,17 @@ object ToolSchemas {
method = "ui.ask",
description = """
Ask the user a question on a small floating card with tappable
- buttons — blocks until they answer (or timeout). Returns which
- button they tapped. The question is also read aloud (TTS), so
- write it as one speakable sentence, no markdown or lists.
+ buttons — blocks until they answer (or timeout). The question is
+ also read aloud (TTS) and the microphone opens afterwards, so the
+ user may simply answer out loud. Write it as one speakable
+ sentence, no markdown or lists.
+
+ The result says how they answered. `button` is the button they
+ tapped or named. If they said something that matches no button,
+ `button` is null and `heard` holds their words: take those as
+ their answer and act on what they mean, which may be a choice, a
+ different option, or "neither". `answered` false means no reply
+ (timeout or dismissed).
USE WHEN the next step genuinely needs the user's choice: which of
several options, yes/no on something irreversible, "要继续吗".
diff --git a/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt b/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt
index 394908d..e4719fe 100644
--- a/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt
+++ b/app/src/main/java/net/kuafuai/andee/ui/CardUi.kt
@@ -92,9 +92,12 @@ object CardUi {
}
class Result(
- /** Which button the user tapped, or null for timeout/superseded/dismissed-elsewhere. */
+ /** Which button the user tapped or named, or null for timeout/superseded/dismissed-elsewhere. */
val button: String?,
- val how: String, // "button" | "timeout" | "superseded" | "tap_outside"
+ /** "button" | "voice" | "voice_other" | "timeout" | "superseded" | "tap_outside" */
+ val how: String,
+ /** The user's own words, when they answered by voice. */
+ val heard: String? = null,
)
/**
@@ -108,9 +111,15 @@ object CardUi {
question: String,
buttons: List<String>,
timeoutMs: Long = AUTO_DISMISS_MS,
+ // True only for a question the brain is blocked on (`ui.ask`). Speech
+ // that names no button then closes the card and travels back in the tool
+ // result for the model to judge. Every other card (a notification's
+ // 看看/忽略, the self-check nudge) has nobody waiting for words, so
+ // unmatched speech must stay an ordinary turn and leave the card alone.
+ captureSpeech: Boolean = false,
onDone: (Result) -> Unit,
) {
- show(question, buttons, cancelable = false, timeoutMs, onDone)
+ show(question, buttons, cancelable = false, timeoutMs, captureSpeech, onDone)
}
/**
@@ -122,7 +131,7 @@ object CardUi {
// same condition one call later — so returning here changes nothing
// observable, it just keeps the label lookup off a null.
val ctx = appContext ?: return
- show(text, listOf(AppLocale.str(ctx, R.string.card_ack)), cancelable = true, timeoutMs, onDone)
+ show(text, listOf(AppLocale.str(ctx, R.string.card_ack)), cancelable = true, timeoutMs, false, onDone)
}
/**
@@ -151,48 +160,76 @@ object CardUi {
main.post { current?.close("superseded") }
}
+ /** What [tryAnswerByVoice] did with a spoken sentence. */
+ enum class VoiceOutcome {
+ /** No card was waiting on speech; the sentence is an ordinary turn. */
+ NOT_HANDLED,
+
+ /** It named a button; the card closed as if that button was tapped. */
+ BUTTON,
+
+ /** A `ui.ask` card was waiting and the words named no button; the card
+ * closed and the words travel back in the tool result for the model. */
+ FORWARDED,
+ }
+
/**
- * Try to answer the current card by matching [spokenText] against its
- * button labels. Returns true if a match was found and the card was closed
- * (the caller should not start a new turn); false if no card is up or no
- * button matched (the caller may proceed with a normal turn).
+ * Answer the card that is up with something the user said.
*
- * Called from the ASR path when the user speaks while a card is visible.
- * The match is case-insensitive and normalised: "是" / "yes" / "确定" all
- * hit a button labelled "是" or "Yes" or "确定". Partial matches (spoken
- * text is a prefix or the button is a prefix of the spoken text) also
- * count, so "好" matches "好的".
+ * A sentence that is (nearly) just a button label presses it. A card
+ * opened with `captureSpeech` (a `ui.ask`, the brain is blocked on it)
+ * treats any other sentence as the answer too: the card closes and the
+ * words go back in the tool result, so the model judges them against the
+ * options it offered. That is why this never starts a turn of its own: a
+ * new turn would queue behind the very tool call this card is blocking.
*
- * Must be called from any thread; callback happens on the main thread.
+ * Matching is deliberately strict. "否则帮我查一下" starts with 否 and is not
+ * a no, so a label only counts when it is most of what was said.
+ *
+ * Blocks the calling thread until the main thread has decided, with no
+ * timeout: the caller is on an ASR worker, and giving up early would let
+ * the same sentence start a second turn after it already closed the card.
*/
- fun tryAnswerByVoice(spokenText: String): Boolean {
- val text = spokenText.trim().lowercase()
- if (text.isEmpty()) return false
+ fun tryAnswerByVoice(spokenText: String): VoiceOutcome {
+ val said = spokenText.trim()
+ if (said.isEmpty()) return VoiceOutcome.NOT_HANDLED
+ if (Looper.myLooper() == Looper.getMainLooper()) return decideVoice(said)
val latch = java.util.concurrent.CountDownLatch(1)
- val matched = java.util.concurrent.atomic.AtomicBoolean(false)
+ var outcome = VoiceOutcome.NOT_HANDLED
main.post {
try {
- val handle = current ?: return@post
- val buttons = handle.buttons
- val hit = buttons.firstOrNull { btn ->
- val label = btn.trim().lowercase()
- label == text || label.startsWith(text) || text.startsWith(label)
- }
- if (hit != null) {
- matched.set(true)
- handle.close("voice", hit)
- }
+ outcome = decideVoice(said)
} finally {
latch.countDown()
}
}
- // Block until the main thread has decided. The caller is on the ASR
- // callback thread and must know whether to bail or proceed before
- // returning. Timeout guards against the main thread being dead.
- latch.await(200, java.util.concurrent.TimeUnit.MILLISECONDS)
- return matched.get()
+ latch.await()
+ return outcome
}
+ private fun decideVoice(said: String): VoiceOutcome {
+ val handle = current ?: return VoiceOutcome.NOT_HANDLED
+ if (handle.finished) return VoiceOutcome.NOT_HANDLED
+ val spoken = normalizeForMatch(said)
+ val hit = handle.buttons.firstOrNull { label ->
+ val l = normalizeForMatch(label)
+ l.isNotEmpty() && (spoken == l || (spoken.contains(l) && l.length * 2 >= spoken.length))
+ }
+ if (hit != null) {
+ handle.close("voice", hit, heard = said)
+ return VoiceOutcome.BUTTON
+ }
+ if (handle.captureSpeech) {
+ handle.close("voice_other", null, heard = said)
+ return VoiceOutcome.FORWARDED
+ }
+ return VoiceOutcome.NOT_HANDLED
+ }
+
+ /** Lower-cased, with spaces and punctuation dropped, so "好的。" equals "好的". */
+ private fun normalizeForMatch(s: String): String =
+ s.lowercase().filter { it.isLetterOrDigit() }
+
// ------------------------------------------------------------------
private class CardHandle(
@@ -201,11 +238,13 @@ object CardUi {
val host: FloatingWindowUi?,
/** Button labels, in the order they were passed to show(). */
val buttons: List<String>,
+ /** Whether speech that names no button should still answer this card. */
+ val captureSpeech: Boolean = false,
@Volatile var finished: Boolean = false,
var onDone: ((Result) -> Unit)? = null,
var timeout: Runnable? = null,
) {
- fun close(how: String, button: String? = null) {
+ fun close(how: String, button: String? = null, heard: String? = null) {
if (finished) return
finished = true
timeout?.let { main.removeCallbacks(it) }
@@ -218,7 +257,7 @@ object CardUi {
// The ball can stop attending — the question is answered, timed
// out, or was replaced by a newer one (which re-arms it below).
ballWindow()?.setHolding(false)
- onDone?.invoke(Result(button, how))
+ onDone?.invoke(Result(button, how, heard))
}
}
@@ -243,6 +282,7 @@ object CardUi {
buttons: List<String>,
cancelable: Boolean,
timeoutMs: Long,
+ captureSpeech: Boolean,
onDone: (Result) -> Unit,
) {
val windowManager = wm ?: return
@@ -393,7 +433,7 @@ object CardUi {
cardReal.tailX = -1
// ---- handle + callbacks ------------------------------------
- val handle = CardHandle(cardReal, host = null, buttons = buttons)
+ val handle = CardHandle(cardReal, host = null, buttons = buttons, captureSpeech = captureSpeech)
handle.onDone = { r -> main.post { onDone(r) } }
wireButtons(handle, btnViews, btnLabels, cancelable, cardReal, title)
val timeoutRunnable = Runnable { handle.close("timeout") }
@@ -431,7 +471,7 @@ object CardUi {
cardReal.tailX = cardReal.measuredWidth / 2
// ---- handle + callbacks ----------------------------------------
- val handle = CardHandle(cardReal, host, buttons = buttons)
+ val handle = CardHandle(cardReal, host, buttons = buttons, captureSpeech = captureSpeech)
handle.onDone = { r -> main.post { onDone(r) } }
wireButtons(handle, btnViews, btnLabels, cancelable, cardReal, title)
val timeoutRunnable = Runnable { handle.close("timeout") }