Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。
feat(desktop): voice on the Mac — speak to Andee, hear it answer
booboosui86f3e6a2fda5
diff --git a/desktop/build.gradle b/desktop/build.gradleindex 9b064a6..6a05fbd 100644--- a/desktop/build.gradle+++ b/desktop/build.gradle@@ -58,6 +58,25 @@ compose.desktop {packageVersion = '1.0.0'macOS {bundleID = 'net.kuafuai.andee.mac'+ entitlementsFile.set(project.file('entitlements.plist'))+ runtimeEntitlementsFile.set(project.file('entitlements.plist'))+ // macOS shows these when Andee first asks; without them the request is refused silently.+ infoPlist {+ extraKeysRawXml = '''+ <key>NSMicrophoneUsageDescription</key>+ <string>Andee listens when you press the microphone button, to hear what you ask.</string>+ <key>NSAppleEventsUsageDescription</key>+ <string>Andee uses Calendar, Reminders, Contacts and other apps when you ask it to.</string>+ <key>NSCalendarsUsageDescription</key>+ <string>Andee reads and adds calendar events when you ask it to.</string>+ <key>NSRemindersUsageDescription</key>+ <string>Andee sets reminders when you ask it to.</string>+ <key>NSContactsUsageDescription</key>+ <string>Andee looks up contacts when you ask it to.</string>+ <key>NSCameraUsageDescription</key>+ <string>Andee looks through the camera only when you ask it to.</string>+ '''+ }}}}diff --git a/desktop/entitlements.plist b/desktop/entitlements.plistnew file mode 100644index 0000000..35830b4--- /dev/null+++ b/desktop/entitlements.plist@@ -0,0 +1,27 @@+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">+<!--+ The hardened runtime refuses these silently unless they are declared here:+ no microphone prompt ever appears, Apple Events to Calendar/Contacts fail,+ and the JVM cannot run. Every key is something Andee does on purpose.+-->+<plist version="1.0">+<dict>+ <!-- The JVM generates code at run time, and JNA loads Andee's own native bridge. -->+ <key>com.apple.security.cs.allow-jit</key>+ <true/>+ <key>com.apple.security.cs.allow-unsigned-executable-memory</key>+ <true/>+ <key>com.apple.security.cs.disable-library-validation</key>+ <true/>+ <!-- Push-to-talk. -->+ <key>com.apple.security.device.audio-input</key>+ <true/>+ <!-- Looking through the camera when asked. -->+ <key>com.apple.security.device.camera</key>+ <true/>+ <!-- Calendar, Reminders, Contacts and other apps, via AppleScript. -->+ <key>com.apple.security.automation.apple-events</key>+ <true/>+</dict>+</plist>diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.ktindex a90850c..e804b04 100644--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt@@ -21,6 +21,9 @@ object Assistant {@Volatile var onBusy: (Boolean) -> Unit = {}@Volatile var onSceneChanged: (String?) -> Unit = {}+ /** Speak answers aloud: on after the user spoke, off after they typed (a typed question wants a quiet answer). */+ @Volatile var speakReplies = false+/*** Shows a command for approval and returns the button pressed. Set by the* window; until it is, everything that needs asking is declined.@@ -105,12 +108,14 @@ object Assistant {Log.i(TAG, "building the brain (${cfg.mode}, ${cfg.model})")val b = LocalBrain(cfg = cfg,- host = DesktopBrainHost(providers(), MacPrompt.Capabilities(shell = true, screen = true)),+ host = DesktopBrainHost(providers(), MacPrompt.Capabilities(shell = true, screen = true, voice = true)),onProgress = { onProgress(it) },onFinal = { text ->- Scrollback.add(Scrollback.Role.ASSISTANT, text)- onFinal(text)+ val clean = net.kuafuai.andee.brain.SpeechMood.parse(text).text.replace("[END]", "").trim()+ Scrollback.add(Scrollback.Role.ASSISTANT, clean)+ onFinal(clean)onBusy(false)+ if (speakReplies) net.kuafuai.andee.desktop.voice.MacVoice.speak(text)},onTurnEnd = { onBusy(false) },)@@ -119,7 +124,8 @@ object Assistant {return b}- fun say(text: String) {+ fun say(text: String, spoken: Boolean = false) {+ speakReplies = spokenval t = text.trim()if (t.isEmpty()) returnScrollback.add(Scrollback.Role.USER, t)@@ -131,6 +137,7 @@ object Assistant {fun stop() {brain?.cancel()shell.stopAll()+ net.kuafuai.andee.desktop.voice.MacVoice.stopSpeaking()onBusy(false)}diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.ktindex 19d4178..728033b 100644--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt@@ -61,6 +61,8 @@ fun ChatScreen(onOpenSettings: () -> Unit) {var progress by remember { mutableStateOf("") }var busy by remember { mutableStateOf(false) }var input by remember { mutableStateOf("") }+ var voice by remember { mutableStateOf(net.kuafuai.andee.desktop.voice.MacVoice.State.IDLE) }+ var heard by remember { mutableStateOf("") }val list = rememberLazyListState()LaunchedEffect(Unit) {@@ -68,6 +70,12 @@ fun ChatScreen(onOpenSettings: () -> Unit) {Assistant.onProgress = { progress = it }Assistant.onFinal = { progress = "" }Assistant.onBusy = { b -> busy = b; if (!b) progress = "" }+ net.kuafuai.andee.desktop.voice.MacVoice.onState = { voice = it }+ net.kuafuai.andee.desktop.voice.MacVoice.onPartial = { heard = it }+ net.kuafuai.andee.desktop.voice.MacVoice.onFinal = { said ->+ heard = ""+ Thread { Assistant.say(said, spoken = true) }.start()+ }}LaunchedEffect(rows.size) { if (rows.isNotEmpty()) list.animateScrollToItem(rows.size - 1) }@@ -86,6 +94,14 @@ fun ChatScreen(onOpenSettings: () -> Unit) {Pill(if (Strings.lang() == "zh") "■ 停止" else "■ Stop") { Assistant.stop() }Spacer(Modifier.width(8.dp))}+ val listening = voice == net.kuafuai.andee.desktop.voice.MacVoice.State.LISTENING+ Pill(if (listening) (if (Strings.lang() == "zh") "● 说完了" else "● Done") else "🎤") {+ val v = net.kuafuai.andee.desktop.voice.MacVoice+ if (!v.configured()) {+ progress = if (Strings.lang() == "zh") "还没配语音 Key:在 ⚙ 设置的「语音」里填上。" else "No voice key yet: add it under Voice in ⚙ Settings."+ } else if (v.isListening()) v.stopListening() else v.startListening()+ }+ Spacer(Modifier.width(8.dp))Pill("⚙") { onOpenSettings() }}Spacer(Modifier.height(12.dp))@@ -100,6 +116,9 @@ fun ChatScreen(onOpenSettings: () -> Unit) {}items(rows, key = { it.id }) { r -> Bubble(r) }}+ if (heard.isNotEmpty()) {+ Text("🎤 $heard", color = Palette.text, fontSize = 14.sp, modifier = Modifier.padding(vertical = 6.dp))+ }if (progress.isNotEmpty()) {Text(progress, color = Palette.secondary, fontSize = 13.sp, modifier = Modifier.padding(vertical = 6.dp))}diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.ktindex ad672ed..2675fdb 100644--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt@@ -38,7 +38,7 @@ What you can do here, right now, is exactly what your tool list contains — notappend("- **You cannot see or operate this Mac's screen yet.** There are no screen tools on this device. Do not offer to open, tap, click, type into or look at anything on the screen, and do not say you did.\n")}if (caps.voice) {- append("- The user may be speaking to you (§6 applies).\n")+ append("- The user may type or speak (the microphone button). A spoken turn is transcribed, so §6 about mishearing applies to it; a typed one is exact. Spoken turns are answered aloud, typed ones are not.\n")} else {append("- **The user types to you here; nothing is transcribed.** §6 about mishearing does not apply: take their characters as written.\n")}diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.ktnew file mode 100644index 0000000..65044ee--- /dev/null+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt@@ -0,0 +1,177 @@+package net.kuafuai.andee.desktop.voice++import net.kuafuai.andee.asr.AsrEngine+import net.kuafuai.andee.asr.GoogleAsr+import net.kuafuai.andee.asr.HuoshanAsr+import net.kuafuai.andee.brain.SpeechMood+import net.kuafuai.andee.config.VoiceSettings+import net.kuafuai.andee.desktop.Settings+import net.kuafuai.andee.tts.GoogleTts+import net.kuafuai.andee.tts.HuoshanTts+import net.kuafuai.andee.tts.TtsEngine+import net.kuafuai.andee.util.Log+import java.util.concurrent.Executors+import java.util.concurrent.atomic.AtomicBoolean+import javax.sound.sampled.AudioFormat+import javax.sound.sampled.AudioSystem+import javax.sound.sampled.SourceDataLine+import javax.sound.sampled.TargetDataLine++/**+ * Voice on the Mac: the phone's own 火山 / Google engines from the shared+ * module, on Java Sound for the microphone and the speaker.+ *+ * Listening is push-to-talk (the window's microphone button, or the shortcut):+ * a laptop has no echo cancellation here, so an always-on wake word would hear+ * Andee's own voice. The phone's wake word stays a phone feature for now.+ */+object MacVoice {++ private const val TAG = "Voice"+ private const val ASR_RATE = 16_000++ @Volatile var onPartial: (String) -> Unit = {}+ @Volatile var onFinal: (String) -> Unit = {}+ @Volatile var onState: (State) -> Unit = {}++ enum class State { IDLE, LISTENING, SPEAKING }++ private val io = Executors.newCachedThreadPool { r -> Thread(r, "voice").apply { isDaemon = true } }+ private val listening = AtomicBoolean(false)+ @Volatile private var mic: TargetDataLine? = null+ @Volatile private var asr: AsrEngine? = null+ @Volatile private var speaker: SourceDataLine? = null+ @Volatile private var tts: TtsEngine? = null++ /** What the phone calls VoiceConfig.load(): the same keys, the same defaults. */+ fun settings(): VoiceSettings = VoiceSettings(+ apiKey = Settings.get("api_key"),+ asrEndpoint = Settings.get("asr_endpoint").ifEmpty { "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" },+ asrResourceId = Settings.get("asr_resource_id").ifEmpty { "volc.bigasr.sauc.duration" },+ asrUid = Settings.uid(),+ ttsEndpoint = Settings.get("tts_endpoint").ifEmpty { "wss://openspeech.bytedance.com/api/v3/tts/bidirection" },+ ttsResourceId = Settings.get("tts_resource_id").ifEmpty { "seed-tts-2.0" },+ ttsSpeaker = Settings.get("tts_speaker").ifEmpty { "zh_female_vv_uranus_bigtts" },+ ttsSampleRate = 24_000,+ voiceProvider = Settings.get("voice_provider").ifEmpty { VoiceSettings.VOICE_HUOSHAN },+ googleApiKey = Settings.get("google_api_key"),+ googleBaseUrl = Settings.get("google_base_url").ifEmpty { "https://aiplatform.googleapis.com" },+ googleProject = Settings.get("google_project"),+ googleLocation = Settings.get("google_location").ifEmpty { "asia-northeast1" },+ googleAsrModel = Settings.get("google_asr_model").ifEmpty { "gemini-live-2.5-flash-preview-native-audio" },+ googleTtsModel = Settings.get("google_tts_model").ifEmpty { "gemini-2.5-flash-preview-tts" },+ googleLanguage = Settings.get("google_language"),+ )++ fun configured(): Boolean = settings().let {+ if (it.voiceProvider == VoiceSettings.VOICE_GOOGLE) it.googleApiKey.isNotBlank() else it.apiKey.isNotBlank()+ }++ // ---- listening -------------------------------------------------------------++ fun isListening() = listening.get()++ /** Start listening; [stopListening] (or the engine's own end-of-turn) delivers [onFinal]. */+ fun startListening() {+ if (!listening.compareAndSet(false, true)) return+ stopSpeaking()+ val cfg = settings()+ val engine: AsrEngine = if (cfg.voiceProvider == VoiceSettings.VOICE_GOOGLE) GoogleAsr(cfg, listener) else HuoshanAsr(cfg, listener)+ asr = engine+ onState(State.LISTENING)+ io.execute {+ runCatching {+ engine.start(ASR_RATE, 1)+ val fmt = AudioFormat(ASR_RATE.toFloat(), 16, 1, true, false)+ val line = AudioSystem.getTargetDataLine(fmt).apply { open(fmt, 6400); start() }+ mic = line+ val buf = ByteArray(3200) // 100 ms+ while (listening.get()) {+ val n = line.read(buf, 0, buf.size)+ if (n > 0) engine.sendAudio(buf.copyOf(n), isLast = false)+ }+ engine.sendAudio(ByteArray(0), isLast = true)+ engine.waitFinished(engine.finalWaitMs)+ }.onFailure {+ Log.e(TAG, "listening failed", it)+ listener.onError(it.message ?: "microphone unavailable")+ }+ runCatching { mic?.stop(); mic?.close() }+ mic = null+ runCatching { engine.close() }+ if (asr === engine) asr = null+ onState(State.IDLE)+ }+ }++ fun stopListening() {+ if (listening.compareAndSet(true, false)) asr?.endOfTurn(bySilence = false)+ }++ private val partial = StringBuilder()++ private val listener = object : AsrEngine.Listener {+ override fun onPartial(text: String) { onPartial.invoke(text) }+ override fun onFinal(text: String) {+ listening.set(false)+ if (text.isNotBlank()) onFinal.invoke(text.trim())+ }+ override fun onError(msg: String) {+ listening.set(false)+ Log.w(TAG, "asr: $msg")+ onPartial.invoke("")+ }+ override fun onClosed() { listening.set(false) }+ }++ // ---- speaking ---------------------------------------------------------------++ /** Speak [raw] (mood markers are stripped first). Returns at once; audio plays in the background. */+ fun speak(raw: String) {+ val text = SpeechMood.parse(raw).text.trim()+ if (text.isEmpty() || !configured()) return+ stopSpeaking()+ val cfg = settings()+ val done = AtomicBoolean(false)+ var engineRef: TtsEngine? = null+ val l = object : TtsEngine.Listener {+ override fun onOpen(sampleRate: Int) {+ // The session is ready only now; text pushed before this is dropped (as on the phone).+ engineRef?.let { it.push(text); it.finish() }+ runCatching {+ val fmt = AudioFormat(sampleRate.toFloat(), 16, 1, true, false)+ speaker = AudioSystem.getSourceDataLine(fmt).apply { open(fmt, sampleRate * 2); start() }+ onState(State.SPEAKING)+ }.onFailure { Log.e(TAG, "speaker unavailable", it) }+ }+ override fun onAudio(pcm: ByteArray) { speaker?.write(pcm, 0, pcm.size) }+ override fun onSentenceStart(text: String) {}+ override fun onSentenceEnd(text: String) {}+ override fun onDone() { finish() }+ override fun onError(msg: String) { Log.w(TAG, "tts: $msg"); finish() }+ override fun onClose(reason: TtsEngine.CloseReason) { finish() }+ fun finish() {+ if (!done.compareAndSet(false, true)) return+ io.execute {+ runCatching { speaker?.drain(); speaker?.close() }+ speaker = null+ onState(State.IDLE)+ }+ }+ }+ val engine: TtsEngine = if (cfg.voiceProvider == VoiceSettings.VOICE_GOOGLE) GoogleTts(cfg, l) else HuoshanTts(cfg, l)+ engineRef = engine+ tts = engine+ io.execute {+ runCatching { engine.open() }+ .onFailure { Log.e(TAG, "speaking failed", it); l.onError(it.message ?: "") }+ }+ }++ fun stopSpeaking() {+ runCatching { tts?.cancel() }+ tts = null+ runCatching { speaker?.stop(); speaker?.flush(); speaker?.close() }+ speaker = null+ }+}