Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(desktop): voice on the Mac — speak to Andee, hear it answer

booboosui86f3e6a2fda5
diff --git a/desktop/build.gradle b/desktop/build.gradle
index 9b064a6..6a05fbd 100644
--- a/desktop/build.gradle
+++ b/desktop/build.gradle
@@ -58,6 +58,25 @@ compose.desktop {
packageVersion = '1.0.0'
macOS {
bundleID = 'net.kuafuai.andee.mac'
+ entitlementsFile.set(project.file('entitlements.plist'))
+ runtimeEntitlementsFile.set(project.file('entitlements.plist'))
+ // macOS shows these when Andee first asks; without them the request is refused silently.
+ infoPlist {
+ extraKeysRawXml = '''
+ <key>NSMicrophoneUsageDescription</key>
+ <string>Andee listens when you press the microphone button, to hear what you ask.</string>
+ <key>NSAppleEventsUsageDescription</key>
+ <string>Andee uses Calendar, Reminders, Contacts and other apps when you ask it to.</string>
+ <key>NSCalendarsUsageDescription</key>
+ <string>Andee reads and adds calendar events when you ask it to.</string>
+ <key>NSRemindersUsageDescription</key>
+ <string>Andee sets reminders when you ask it to.</string>
+ <key>NSContactsUsageDescription</key>
+ <string>Andee looks up contacts when you ask it to.</string>
+ <key>NSCameraUsageDescription</key>
+ <string>Andee looks through the camera only when you ask it to.</string>
+ '''
+ }
}
}
}
diff --git a/desktop/entitlements.plist b/desktop/entitlements.plist
new file mode 100644
index 0000000..35830b4
--- /dev/null
+++ b/desktop/entitlements.plist
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!--
+ The hardened runtime refuses these silently unless they are declared here:
+ no microphone prompt ever appears, Apple Events to Calendar/Contacts fail,
+ and the JVM cannot run. Every key is something Andee does on purpose.
+-->
+<plist version="1.0">
+<dict>
+ <!-- The JVM generates code at run time, and JNA loads Andee's own native bridge. -->
+ <key>com.apple.security.cs.allow-jit</key>
+ <true/>
+ <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+ <true/>
+ <key>com.apple.security.cs.disable-library-validation</key>
+ <true/>
+ <!-- Push-to-talk. -->
+ <key>com.apple.security.device.audio-input</key>
+ <true/>
+ <!-- Looking through the camera when asked. -->
+ <key>com.apple.security.device.camera</key>
+ <true/>
+ <!-- Calendar, Reminders, Contacts and other apps, via AppleScript. -->
+ <key>com.apple.security.automation.apple-events</key>
+ <true/>
+</dict>
+</plist>
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
index a90850c..e804b04 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
@@ -21,6 +21,9 @@ object Assistant {
@Volatile var onBusy: (Boolean) -> Unit = {}
@Volatile var onSceneChanged: (String?) -> Unit = {}
+ /** Speak answers aloud: on after the user spoke, off after they typed (a typed question wants a quiet answer). */
+ @Volatile var speakReplies = false
+
/**
* Shows a command for approval and returns the button pressed. Set by the
* window; until it is, everything that needs asking is declined.
@@ -105,12 +108,14 @@ object Assistant {
Log.i(TAG, "building the brain (${cfg.mode}, ${cfg.model})")
val b = LocalBrain(
cfg = cfg,
- host = DesktopBrainHost(providers(), MacPrompt.Capabilities(shell = true, screen = true)),
+ host = DesktopBrainHost(providers(), MacPrompt.Capabilities(shell = true, screen = true, voice = true)),
onProgress = { onProgress(it) },
onFinal = { text ->
- Scrollback.add(Scrollback.Role.ASSISTANT, text)
- onFinal(text)
+ val clean = net.kuafuai.andee.brain.SpeechMood.parse(text).text.replace("[END]", "").trim()
+ Scrollback.add(Scrollback.Role.ASSISTANT, clean)
+ onFinal(clean)
onBusy(false)
+ if (speakReplies) net.kuafuai.andee.desktop.voice.MacVoice.speak(text)
},
onTurnEnd = { onBusy(false) },
)
@@ -119,7 +124,8 @@ object Assistant {
return b
}
- fun say(text: String) {
+ fun say(text: String, spoken: Boolean = false) {
+ speakReplies = spoken
val t = text.trim()
if (t.isEmpty()) return
Scrollback.add(Scrollback.Role.USER, t)
@@ -131,6 +137,7 @@ object Assistant {
fun stop() {
brain?.cancel()
shell.stopAll()
+ net.kuafuai.andee.desktop.voice.MacVoice.stopSpeaking()
onBusy(false)
}
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
index 19d4178..728033b 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
@@ -61,6 +61,8 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
var progress by remember { mutableStateOf("") }
var busy by remember { mutableStateOf(false) }
var input by remember { mutableStateOf("") }
+ var voice by remember { mutableStateOf(net.kuafuai.andee.desktop.voice.MacVoice.State.IDLE) }
+ var heard by remember { mutableStateOf("") }
val list = rememberLazyListState()
LaunchedEffect(Unit) {
@@ -68,6 +70,12 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
Assistant.onProgress = { progress = it }
Assistant.onFinal = { progress = "" }
Assistant.onBusy = { b -> busy = b; if (!b) progress = "" }
+ net.kuafuai.andee.desktop.voice.MacVoice.onState = { voice = it }
+ net.kuafuai.andee.desktop.voice.MacVoice.onPartial = { heard = it }
+ net.kuafuai.andee.desktop.voice.MacVoice.onFinal = { said ->
+ heard = ""
+ Thread { Assistant.say(said, spoken = true) }.start()
+ }
}
LaunchedEffect(rows.size) { if (rows.isNotEmpty()) list.animateScrollToItem(rows.size - 1) }
@@ -86,6 +94,14 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
Pill(if (Strings.lang() == "zh") "■ 停止" else "■ Stop") { Assistant.stop() }
Spacer(Modifier.width(8.dp))
}
+ val listening = voice == net.kuafuai.andee.desktop.voice.MacVoice.State.LISTENING
+ Pill(if (listening) (if (Strings.lang() == "zh") "● 说完了" else "● Done") else "🎤") {
+ val v = net.kuafuai.andee.desktop.voice.MacVoice
+ if (!v.configured()) {
+ progress = if (Strings.lang() == "zh") "还没配语音 Key:在 ⚙ 设置的「语音」里填上。" else "No voice key yet: add it under Voice in ⚙ Settings."
+ } else if (v.isListening()) v.stopListening() else v.startListening()
+ }
+ Spacer(Modifier.width(8.dp))
Pill("⚙") { onOpenSettings() }
}
Spacer(Modifier.height(12.dp))
@@ -100,6 +116,9 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
}
items(rows, key = { it.id }) { r -> Bubble(r) }
}
+ if (heard.isNotEmpty()) {
+ Text("🎤 $heard", color = Palette.text, fontSize = 14.sp, modifier = Modifier.padding(vertical = 6.dp))
+ }
if (progress.isNotEmpty()) {
Text(progress, color = Palette.secondary, fontSize = 13.sp, modifier = Modifier.padding(vertical = 6.dp))
}
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt
index ad672ed..2675fdb 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacPrompt.kt
@@ -38,7 +38,7 @@ What you can do here, right now, is exactly what your tool list contains — not
append("- **You cannot see or operate this Mac's screen yet.** There are no screen tools on this device. Do not offer to open, tap, click, type into or look at anything on the screen, and do not say you did.\n")
}
if (caps.voice) {
- append("- The user may be speaking to you (§6 applies).\n")
+ append("- The user may type or speak (the microphone button). A spoken turn is transcribed, so §6 about mishearing applies to it; a typed one is exact. Spoken turns are answered aloud, typed ones are not.\n")
} else {
append("- **The user types to you here; nothing is transcribed.** §6 about mishearing does not apply: take their characters as written.\n")
}
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt
new file mode 100644
index 0000000..65044ee
--- /dev/null
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt
@@ -0,0 +1,177 @@
+package net.kuafuai.andee.desktop.voice
+
+import net.kuafuai.andee.asr.AsrEngine
+import net.kuafuai.andee.asr.GoogleAsr
+import net.kuafuai.andee.asr.HuoshanAsr
+import net.kuafuai.andee.brain.SpeechMood
+import net.kuafuai.andee.config.VoiceSettings
+import net.kuafuai.andee.desktop.Settings
+import net.kuafuai.andee.tts.GoogleTts
+import net.kuafuai.andee.tts.HuoshanTts
+import net.kuafuai.andee.tts.TtsEngine
+import net.kuafuai.andee.util.Log
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicBoolean
+import javax.sound.sampled.AudioFormat
+import javax.sound.sampled.AudioSystem
+import javax.sound.sampled.SourceDataLine
+import javax.sound.sampled.TargetDataLine
+
+/**
+ * Voice on the Mac: the phone's own 火山 / Google engines from the shared
+ * module, on Java Sound for the microphone and the speaker.
+ *
+ * Listening is push-to-talk (the window's microphone button, or the shortcut):
+ * a laptop has no echo cancellation here, so an always-on wake word would hear
+ * Andee's own voice. The phone's wake word stays a phone feature for now.
+ */
+object MacVoice {
+
+ private const val TAG = "Voice"
+ private const val ASR_RATE = 16_000
+
+ @Volatile var onPartial: (String) -> Unit = {}
+ @Volatile var onFinal: (String) -> Unit = {}
+ @Volatile var onState: (State) -> Unit = {}
+
+ enum class State { IDLE, LISTENING, SPEAKING }
+
+ private val io = Executors.newCachedThreadPool { r -> Thread(r, "voice").apply { isDaemon = true } }
+ private val listening = AtomicBoolean(false)
+ @Volatile private var mic: TargetDataLine? = null
+ @Volatile private var asr: AsrEngine? = null
+ @Volatile private var speaker: SourceDataLine? = null
+ @Volatile private var tts: TtsEngine? = null
+
+ /** What the phone calls VoiceConfig.load(): the same keys, the same defaults. */
+ fun settings(): VoiceSettings = VoiceSettings(
+ apiKey = Settings.get("api_key"),
+ asrEndpoint = Settings.get("asr_endpoint").ifEmpty { "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" },
+ asrResourceId = Settings.get("asr_resource_id").ifEmpty { "volc.bigasr.sauc.duration" },
+ asrUid = Settings.uid(),
+ ttsEndpoint = Settings.get("tts_endpoint").ifEmpty { "wss://openspeech.bytedance.com/api/v3/tts/bidirection" },
+ ttsResourceId = Settings.get("tts_resource_id").ifEmpty { "seed-tts-2.0" },
+ ttsSpeaker = Settings.get("tts_speaker").ifEmpty { "zh_female_vv_uranus_bigtts" },
+ ttsSampleRate = 24_000,
+ voiceProvider = Settings.get("voice_provider").ifEmpty { VoiceSettings.VOICE_HUOSHAN },
+ googleApiKey = Settings.get("google_api_key"),
+ googleBaseUrl = Settings.get("google_base_url").ifEmpty { "https://aiplatform.googleapis.com" },
+ googleProject = Settings.get("google_project"),
+ googleLocation = Settings.get("google_location").ifEmpty { "asia-northeast1" },
+ googleAsrModel = Settings.get("google_asr_model").ifEmpty { "gemini-live-2.5-flash-preview-native-audio" },
+ googleTtsModel = Settings.get("google_tts_model").ifEmpty { "gemini-2.5-flash-preview-tts" },
+ googleLanguage = Settings.get("google_language"),
+ )
+
+ fun configured(): Boolean = settings().let {
+ if (it.voiceProvider == VoiceSettings.VOICE_GOOGLE) it.googleApiKey.isNotBlank() else it.apiKey.isNotBlank()
+ }
+
+ // ---- listening -------------------------------------------------------------
+
+ fun isListening() = listening.get()
+
+ /** Start listening; [stopListening] (or the engine's own end-of-turn) delivers [onFinal]. */
+ fun startListening() {
+ if (!listening.compareAndSet(false, true)) return
+ stopSpeaking()
+ val cfg = settings()
+ val engine: AsrEngine = if (cfg.voiceProvider == VoiceSettings.VOICE_GOOGLE) GoogleAsr(cfg, listener) else HuoshanAsr(cfg, listener)
+ asr = engine
+ onState(State.LISTENING)
+ io.execute {
+ runCatching {
+ engine.start(ASR_RATE, 1)
+ val fmt = AudioFormat(ASR_RATE.toFloat(), 16, 1, true, false)
+ val line = AudioSystem.getTargetDataLine(fmt).apply { open(fmt, 6400); start() }
+ mic = line
+ val buf = ByteArray(3200) // 100 ms
+ while (listening.get()) {
+ val n = line.read(buf, 0, buf.size)
+ if (n > 0) engine.sendAudio(buf.copyOf(n), isLast = false)
+ }
+ engine.sendAudio(ByteArray(0), isLast = true)
+ engine.waitFinished(engine.finalWaitMs)
+ }.onFailure {
+ Log.e(TAG, "listening failed", it)
+ listener.onError(it.message ?: "microphone unavailable")
+ }
+ runCatching { mic?.stop(); mic?.close() }
+ mic = null
+ runCatching { engine.close() }
+ if (asr === engine) asr = null
+ onState(State.IDLE)
+ }
+ }
+
+ fun stopListening() {
+ if (listening.compareAndSet(true, false)) asr?.endOfTurn(bySilence = false)
+ }
+
+ private val partial = StringBuilder()
+
+ private val listener = object : AsrEngine.Listener {
+ override fun onPartial(text: String) { onPartial.invoke(text) }
+ override fun onFinal(text: String) {
+ listening.set(false)
+ if (text.isNotBlank()) onFinal.invoke(text.trim())
+ }
+ override fun onError(msg: String) {
+ listening.set(false)
+ Log.w(TAG, "asr: $msg")
+ onPartial.invoke("")
+ }
+ override fun onClosed() { listening.set(false) }
+ }
+
+ // ---- speaking ---------------------------------------------------------------
+
+ /** Speak [raw] (mood markers are stripped first). Returns at once; audio plays in the background. */
+ fun speak(raw: String) {
+ val text = SpeechMood.parse(raw).text.trim()
+ if (text.isEmpty() || !configured()) return
+ stopSpeaking()
+ val cfg = settings()
+ val done = AtomicBoolean(false)
+ var engineRef: TtsEngine? = null
+ val l = object : TtsEngine.Listener {
+ override fun onOpen(sampleRate: Int) {
+ // The session is ready only now; text pushed before this is dropped (as on the phone).
+ engineRef?.let { it.push(text); it.finish() }
+ runCatching {
+ val fmt = AudioFormat(sampleRate.toFloat(), 16, 1, true, false)
+ speaker = AudioSystem.getSourceDataLine(fmt).apply { open(fmt, sampleRate * 2); start() }
+ onState(State.SPEAKING)
+ }.onFailure { Log.e(TAG, "speaker unavailable", it) }
+ }
+ override fun onAudio(pcm: ByteArray) { speaker?.write(pcm, 0, pcm.size) }
+ override fun onSentenceStart(text: String) {}
+ override fun onSentenceEnd(text: String) {}
+ override fun onDone() { finish() }
+ override fun onError(msg: String) { Log.w(TAG, "tts: $msg"); finish() }
+ override fun onClose(reason: TtsEngine.CloseReason) { finish() }
+ fun finish() {
+ if (!done.compareAndSet(false, true)) return
+ io.execute {
+ runCatching { speaker?.drain(); speaker?.close() }
+ speaker = null
+ onState(State.IDLE)
+ }
+ }
+ }
+ val engine: TtsEngine = if (cfg.voiceProvider == VoiceSettings.VOICE_GOOGLE) GoogleTts(cfg, l) else HuoshanTts(cfg, l)
+ engineRef = engine
+ tts = engine
+ io.execute {
+ runCatching { engine.open() }
+ .onFailure { Log.e(TAG, "speaking failed", it); l.onError(it.message ?: "") }
+ }
+ }
+
+ fun stopSpeaking() {
+ runCatching { tts?.cancel() }
+ tts = null
+ runCatching { speaker?.stop(); speaker?.flush(); speaker?.close() }
+ speaker = null
+ }
+}