Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(desktop): meeting recorder — hold the mic, hand back a transcript

booboosui7ec9697f2bd7
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
index 34132e2..29e48ed 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Assistant.kt
@@ -83,6 +83,7 @@ object Assistant {
val notes = NotebookTools { onSceneChanged(it) }
return listOf(
UiTools(),
+ MeetingTools(),
object : DesktopBrainHost.ToolProvider {
override fun handles(method: String) = notes.handles(method)
override fun dispatch(method: String, p: JSONObject, silent: Boolean, untrusted: Boolean) =
@@ -170,6 +171,14 @@ object Assistant {
}
fun start() {
+ Meeting.onUserEnded = { payload ->
+ // The user ended the recording themselves, so the brain does not have the
+ // transcript — ask it for the minutes on a fresh turn (the phone's asr.final path).
+ busy(true)
+ speakReplies = false
+ ensure().submit(meetingMinutesQuery(payload))
+ }
+ Meeting.onRecordingChanged = { MacBall.onRecording(it) }
net.kuafuai.andee.desktop.mac.MacNotifications.start(notifyHost)
DesktopScheduler.onDue = { todo -> busy(true); ensure().wake(dueText(todo), untrusted = todo.untrusted) }
DesktopScheduler.catchUp()
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
index 0dff5ac..b55ab41 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/ChatScreen.kt
@@ -64,11 +64,13 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
var voice by remember { mutableStateOf(net.kuafuai.andee.desktop.voice.MacVoice.State.IDLE) }
var heard by remember { mutableStateOf("") }
var artifacts by remember { mutableStateOf(Pages.artifacts()) }
+ var recording by remember { mutableStateOf(Meeting.isActive()) }
val list = rememberLazyListState()
LaunchedEffect(Unit) {
Scrollback.addListener { rows = Scrollback.snapshot() }
Pages.onChange = { artifacts = Pages.artifacts() }
+ Meeting.onRecordingChanged = { recording = it }
Assistant.onProgress = { progress = it }
Assistant.onFinal = { progress = "" }
Assistant.onBusy = { b -> busy = b; if (!b) progress = "" }
@@ -97,7 +99,10 @@ fun ChatScreen(onOpenSettings: () -> Unit) {
Row(verticalAlignment = Alignment.CenterVertically) {
Text("Andee", color = Palette.text, fontSize = 20.sp)
Spacer(Modifier.weight(1f))
- if (busy) {
+ if (recording) {
+ Pill(if (Strings.lang() == "zh") "■ 停止录音" else "■ Stop recording") { Meeting.stopFromUser() }
+ Spacer(Modifier.width(8.dp))
+ } else if (busy) {
Pill(if (Strings.lang() == "zh") "■ 停止" else "■ Stop") { Assistant.stop() }
Spacer(Modifier.width(8.dp))
}
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacBall.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacBall.kt
index 3fe538d..de15ead 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacBall.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/MacBall.kt
@@ -40,6 +40,7 @@ object MacBall {
@Volatile private var renderer: BallRenderer? = null
@Volatile private var voice = MacVoice.State.IDLE
@Volatile private var busy = false
+ @Volatile private var recording = false
@Volatile private var emotion: Mood? = null
/** The face the model asked for in the reply about to be spoken. */
@@ -69,6 +70,12 @@ object MacBall {
apply()
}
+ /** A meeting is being recorded: the ball is listening, whatever else is happening. */
+ fun onRecording(b: Boolean) {
+ recording = b
+ apply()
+ }
+
/** A tool ran: a nod, then back to the working face on its own. */
fun onTool() { renderer?.pulseTool() }
@@ -95,6 +102,7 @@ object MacBall {
private fun apply() {
val r = renderer ?: return
val mood = when {
+ recording -> Mood.LISTENING
voice == MacVoice.State.LISTENING -> Mood.LISTENING
voice == MacVoice.State.SPEAKING -> Mood.SPEAKING
busy -> Mood.THINKING
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Main.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Main.kt
index 6d8e06e..0fffed5 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Main.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Main.kt
@@ -35,7 +35,10 @@ fun main() {
Assistant.hideWindows = { hide -> net.kuafuai.andee.desktop.mac.Bridge.setOwnAlpha(if (hide) 0.0 else 1.0) }
FloatingBall(
- onTap = { if (!net.kuafuai.andee.desktop.voice.MacVoice.toggle()) { chat = true; settings = true } },
+ onTap = {
+ if (Meeting.isActive()) Meeting.stopFromUser()
+ else if (!net.kuafuai.andee.desktop.voice.MacVoice.toggle()) { chat = true; settings = true }
+ },
onLongPress = { chat = !chat },
onWindow = { ballWindow = it },
)
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Meeting.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Meeting.kt
new file mode 100644
index 0000000..5b3ee0b
--- /dev/null
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/Meeting.kt
@@ -0,0 +1,279 @@
+package net.kuafuai.andee.desktop
+
+import net.kuafuai.andee.asr.AsrEngine
+import net.kuafuai.andee.asr.GoogleAsr
+import net.kuafuai.andee.asr.HuoshanAsr
+import net.kuafuai.andee.config.VoiceSettings
+import net.kuafuai.andee.desktop.voice.MacVoice
+import net.kuafuai.andee.util.Log
+import org.json.JSONObject
+import java.io.File
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.Locale
+import java.util.TreeMap
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
+import javax.sound.sampled.AudioFormat
+import javax.sound.sampled.AudioSystem
+
+/**
+ * Hold the microphone for a whole meeting and hand back a verbatim transcript.
+ * The phone's MeetingRecorder, on the Mac's Java Sound: same segment rotation,
+ * same ordered-final flush, same no-summary split (the brain writes the minutes
+ * and lays them out with show_html).
+ *
+ * Not runtime-tested here: the audio path needs a live microphone and a voice
+ * key, which this environment cannot drive. It compiles and registers; the
+ * real-meeting behaviour is a device test.
+ */
+object Meeting {
+
+ private const val TAG = "Meeting"
+ private const val ASR_RATE = 16_000
+ private const val SEGMENT_SOFT_MS = 120_000L
+ private const val SEGMENT_HARD_MS = 180_000L
+ private const val PAUSE_MS = 800L
+ private const val MAX_RETURN_CHARS = 60_000
+
+ private const val BY_USER = "user"
+ private const val BY_BRAIN = "brain"
+ private const val BY_SHUTDOWN = "shutdown"
+
+ /** Called when the user (not the brain) ends a recording, with the result payload; the host writes the minutes. */
+ @Volatile var onUserEnded: (JSONObject) -> Unit = {}
+
+ @Volatile var onRecordingChanged: (Boolean) -> Unit = {}
+
+ private val exec = Executors.newSingleThreadExecutor { r -> Thread(r, "meeting").apply { isDaemon = true } }
+ private val io = Executors.newSingleThreadExecutor { r -> Thread(r, "meeting-writer").apply { isDaemon = true } }
+
+ private val recording = AtomicBoolean(false)
+ @Volatile private var active: Segment? = null
+ @Volatile private var rotatePending = false
+ private val nextIndex = AtomicInteger(0)
+ private var title = ""
+ private var startedAt = 0L
+ @Volatile private var file: File? = null
+ private val transcript = StringBuilder()
+ private val finals = TreeMap<Int, String>()
+ private var nextToFlush = 0
+
+ fun isActive(): Boolean = recording.get()
+
+ fun start(name: String?): JSONObject {
+ if (recording.get()) throw IllegalStateException("a meeting is already being recorded — stop_meeting first")
+ if (MacVoice.isListening()) throw IllegalStateException("the microphone is busy with a voice turn; try again in a moment")
+ val cfg = MacVoice.settings()
+ if (cfg.apiKey.isEmpty() && cfg.googleApiKey.isEmpty()) {
+ throw IllegalStateException("no voice key is set, so there is nothing to transcribe with — the meeting would be silent")
+ }
+ title = name?.trim().orEmpty().ifEmpty { if (Strings.lang() == "zh") "会议" else "Meeting" }
+ startedAt = System.currentTimeMillis()
+ nextIndex.set(0)
+ nextToFlush = 0
+ synchronized(finals) { finals.clear() }
+ synchronized(transcript) { transcript.setLength(0) }
+ file = openTranscriptFile()
+ val first = runCatching { openSegment() }.getOrElse {
+ throw IllegalStateException("could not start the recogniser: ${it.message}", it)
+ }
+ active = first
+ recording.set(true)
+ exec.execute { micLoop() }
+ onRecordingChanged(true)
+ Scrollback.add(Scrollback.Role.ASSISTANT, "📼 " + (if (Strings.lang() == "zh") "开始录音了。" else "Recording started."))
+ Log.i(TAG, "meeting started: $title")
+ return JSONObject().put("recording", true).put("title", title).put("started_at", startedAt)
+ }
+
+ /** The brain's own stop_meeting: the transcript is already in the tool result, so no minutes are asked for. */
+ fun stop(): JSONObject {
+ if (!recording.compareAndSet(true, false)) throw IllegalStateException("no meeting is being recorded")
+ return stopImpl(BY_BRAIN)
+ }
+
+ /** The user's stop: end the recording and hand the transcript to the brain for minutes. */
+ fun stopFromUser() {
+ if (!recording.compareAndSet(true, false)) return
+ exec.execute {
+ runCatching { stopImpl(BY_USER) }.getOrNull()?.let { runCatching { onUserEnded(it) } }
+ }
+ }
+
+ fun status(): JSONObject = JSONObject()
+ .put("recording", recording.get())
+ .put("title", title)
+ .put("started_at", startedAt)
+ .put("elapsed_ms", if (startedAt == 0L) 0L else System.currentTimeMillis() - startedAt)
+ .put("segments", nextIndex.get())
+ .put("chars", synchronized(transcript) { transcript.length })
+
+ fun shutdown() {
+ if (recording.compareAndSet(true, false)) runCatching { stopImpl(BY_SHUTDOWN) }
+ exec.shutdown()
+ io.shutdown()
+ onRecordingChanged(false)
+ }
+
+ private fun stopImpl(stoppedBy: String): JSONObject {
+ val seg = active
+ active = null
+ if (seg != null) retire(seg)
+ onRecordingChanged(false)
+ Scrollback.add(Scrollback.Role.ASSISTANT, "📼 " + (if (Strings.lang() == "zh") "录音结束了。" else "Recording ended."))
+ return result(stoppedBy)
+ }
+
+ // ---- audio --------------------------------------------------------------
+
+ private fun micLoop() {
+ runCatching {
+ val fmt = AudioFormat(ASR_RATE.toFloat(), 16, 1, true, false)
+ val line = AudioSystem.getTargetDataLine(fmt).apply { open(fmt, ASR_RATE * 2); start() }
+ val buf = ByteArray(3200) // 100 ms
+ while (recording.get()) {
+ val n = line.read(buf, 0, buf.size)
+ if (n > 0) onPcm(buf.copyOf(n))
+ }
+ runCatching { line.stop(); line.close() }
+ }.onFailure { Log.e(TAG, "microphone loop failed", it) }
+ }
+
+ private fun onPcm(chunk: ByteArray) {
+ val seg = active ?: return
+ runCatching { seg.send(chunk) }.onFailure { Log.w(TAG, "sendAudio failed on segment ${seg.index}", it) }
+ maybeRotate(seg)
+ }
+
+ private fun maybeRotate(seg: Segment) {
+ if (rotatePending || !recording.get()) return
+ val now = System.currentTimeMillis()
+ val age = now - seg.openedAt
+ if (age < SEGMENT_SOFT_MS) return
+ if (age < SEGMENT_HARD_MS && now - seg.lastTextAt < PAUSE_MS) return
+ rotatePending = true
+ runCatching { exec.execute { rotate(seg) } }.onFailure { rotatePending = false }
+ }
+
+ private fun rotate(old: Segment) {
+ try {
+ if (!recording.get() || active !== old) return
+ val fresh = openSegment()
+ active = fresh
+ retire(old)
+ } catch (t: Throwable) {
+ Log.w(TAG, "rotation failed, keeping segment ${old.index}", t)
+ old.openedAt = System.currentTimeMillis()
+ } finally {
+ rotatePending = false
+ }
+ }
+
+ private fun openSegment(): Segment {
+ val seg = Segment(nextIndex.getAndIncrement(), MacVoice.settings())
+ seg.asr.start(ASR_RATE, 1)
+ return seg
+ }
+
+ private fun retire(seg: Segment, waitForFinal: Boolean = true) {
+ runCatching { seg.asr.sendAudio(ByteArray(0), isLast = true) }
+ if (waitForFinal) runCatching { seg.asr.waitFinished(seg.asr.finalWaitMs) }
+ runCatching { seg.asr.close() }
+ seg.settle()
+ }
+
+ // ---- transcript ---------------------------------------------------------
+
+ private fun noteFinal(index: Int, text: String) {
+ val ready = ArrayList<String>()
+ synchronized(finals) {
+ finals[index] = text
+ while (finals.containsKey(nextToFlush)) {
+ val t = finals.remove(nextToFlush).orEmpty().trim()
+ nextToFlush++
+ if (t.isNotEmpty()) ready.add(t)
+ }
+ }
+ if (ready.isEmpty()) return
+ val block = ready.joinToString("\n", postfix = "\n")
+ synchronized(transcript) { transcript.append(block) }
+ val f = file ?: return
+ runCatching { io.execute { runCatching { f.appendText(block) }.onFailure { Log.w(TAG, "transcript append failed", it) } } }
+ }
+
+ private fun openTranscriptFile(): File? = runCatching {
+ val dir = File(AppDirs.root, "meetings").apply { mkdirs() }
+ val stamp = SimpleDateFormat("yyyyMMdd-HHmmss", Locale.US).format(startedAt)
+ File(dir, "meeting-$stamp.txt").apply {
+ writeText("# $title\n# ${SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US).format(startedAt)}\n\n")
+ }
+ }.onFailure { Log.w(TAG, "could not open transcript file", it) }.getOrNull()
+
+ private fun result(stoppedBy: String): JSONObject {
+ val full = synchronized(transcript) { transcript.toString() }.trim()
+ val truncated = full.length > MAX_RETURN_CHARS
+ return JSONObject()
+ .put("ended", true)
+ .put("title", title)
+ .put("duration_ms", System.currentTimeMillis() - startedAt)
+ .put("segments", nextIndex.get())
+ .put("chars", full.length)
+ .put("file", file?.let { "meetings/${it.name}" } ?: JSONObject.NULL)
+ .put("stopped_by", stoppedBy)
+ .put("truncated", truncated)
+ .put("transcript", if (truncated) elide(full) else full)
+ }
+
+ private fun elide(full: String): String {
+ val half = MAX_RETURN_CHARS / 2
+ return full.take(half) + "\n\n[${full.length - MAX_RETURN_CHARS} characters elided in the middle; the full transcript is on disk]\n\n" + full.takeLast(half)
+ }
+
+ // ---- one recogniser session ---------------------------------------------
+
+ private class Segment(val index: Int, cfg: VoiceSettings) : AsrEngine.Listener {
+ val asr: AsrEngine = if (cfg.voiceProvider == VoiceSettings.VOICE_GOOGLE) GoogleAsr(cfg, this) else HuoshanAsr(cfg, this)
+ @Volatile var openedAt = System.currentTimeMillis()
+ @Volatile var lastTextAt = System.currentTimeMillis()
+ private var partial = ""
+ private val settled = AtomicBoolean(false)
+
+ fun send(pcm: ByteArray) = asr.sendAudio(pcm, isLast = false)
+ fun settle(text: String = partial) { if (settled.compareAndSet(false, true)) Meeting.noteFinal(index, text) }
+ override fun onPartial(text: String) {
+ if (text.isEmpty() || text == partial) return
+ partial = text
+ lastTextAt = System.currentTimeMillis()
+ }
+ override fun onFinal(text: String) = settle(text)
+ override fun onError(msg: String) { Log.w(TAG, "segment $index error: $msg"); settle() }
+ override fun onClosed() {}
+ }
+}
+
+/** meeting.start / meeting.status / meeting.stop, so the brain sees them. */
+class MeetingTools : DesktopBrainHost.ToolProvider {
+ override fun handles(method: String) = method.startsWith("meeting.")
+ override fun dispatch(method: String, p: JSONObject, silent: Boolean, untrusted: Boolean): Any? = when (method) {
+ "meeting.start" -> Meeting.start(p.optString("title").ifEmpty { null })
+ "meeting.status" -> Meeting.status()
+ "meeting.stop" -> Meeting.stop()
+ else -> throw IllegalArgumentException("unknown method $method")
+ }
+}
+
+/** The minutes the device asks the brain for when the user ends the recording themselves. */
+fun meetingMinutesQuery(p: JSONObject): String {
+ val title = p.optString("title").ifEmpty { if (Strings.lang() == "zh") "会议" else "Meeting" }
+ val minutes = (p.optLong("duration_ms") / 60_000).coerceAtLeast(1)
+ return buildString {
+ append("(the user tapped to end the recording) The meeting just finished, and the transcript is below.\n")
+ append("Turn it into meeting minutes, then use the show_html tool to lay the minutes out as an HTML page and show it to the user — a text-only reply is something they cannot see.\n")
+ append("The minutes should contain: the subject, the key points, the conclusions reached, and action items (who does what).")
+ append("The transcript comes out of speech recognition, so it will have wrong characters and missing punctuation; follow the context.\n\n")
+ append("Title: $title\nDuration: about $minutes minutes\n\nTranscript:\n")
+ append(p.optString("transcript"))
+ }
+}
diff --git a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt
index 63b78a4..2030e05 100644
--- a/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt
+++ b/desktop/src/main/kotlin/net/kuafuai/andee/desktop/voice/MacVoice.kt
@@ -79,6 +79,7 @@ object MacVoice {
/** Start listening; [stopListening] (or the engine's own end-of-turn) delivers [onFinal]. */
fun startListening() {
+ if (net.kuafuai.andee.desktop.Meeting.isActive()) return // the meeting holds the microphone
if (!listening.compareAndSet(false, true)) return
stopSpeaking()
val cfg = settings()
@@ -117,6 +118,8 @@ object MacVoice {
return true
}
+ fun recordingMeeting(): Boolean = net.kuafuai.andee.desktop.Meeting.isActive()
+
fun stopListening() {
if (listening.compareAndSet(true, false)) asr?.endOfTurn(bySilence = false)
}
@@ -141,6 +144,7 @@ object MacVoice {
/** Speak [raw] (mood markers are stripped first). Returns at once; audio plays in the background. */
fun speak(raw: String) {
+ if (net.kuafuai.andee.desktop.Meeting.isActive()) return // never speak into the recording
val text = SpeechMood.parse(raw).text.trim()
if (text.isEmpty() || !configured()) return
stopSpeaking()