Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat(optimus): start from Mercury, live transcript, voice levels
Salman Qureshi90646fd54722
diff --git a/plugins/optimus/companion/src/audio.rs b/plugins/optimus/companion/src/audio.rsindex a4d2d00d..c2d295bb 100644--- a/plugins/optimus/companion/src/audio.rs+++ b/plugins/optimus/companion/src/audio.rs@@ -9,7 +9,7 @@//! Mercury talks.use std::collections::VecDeque;-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -28,6 +28,8 @@ pub struct Output {start: Instant,/// Milliseconds since `start` when the speaker last played sound.last_sound: AtomicU64,+ /// Loudest sample played since the last `take_level` (f32 bits), for the app's call circles.+ peak: AtomicU32,}impl Output {@@ -99,10 +101,19 @@ impl Output {}}+ /// How loud the speaker was since the last call (0-1), then start over.+ pub fn take_level(&self) -> f32 {+ f32::from_bits(self.peak.swap(0, Ordering::Relaxed))+ }+fn next(&self) -> f32 {let s = self.queue.lock().unwrap().pop_front();match s {Some(v) => {+ let a = v.abs();+ if a > f32::from_bits(self.peak.load(Ordering::Relaxed)) {+ self.peak.store(a.to_bits(), Ordering::Relaxed);+ }// Only real sound (speech, chimes) counts: the quiet thinking tick doesn't mute the mic.if v.abs() > 0.06 {self.last_sound.store(self.start.elapsed().as_millis() as u64, Ordering::Relaxed);@@ -159,6 +170,7 @@ pub fn open() -> Result<Audio, String> {rate: out_cfg.sample_rate().0,start: Instant::now(),last_sound: AtomicU64::new(0),+ peak: AtomicU32::new(0),});let in_stream = build_input(&input, &in_cfg, mic_tx)?;let out_stream = build_output(&output, &out_cfg, out.clone())?;diff --git a/plugins/optimus/companion/src/core.rs b/plugins/optimus/companion/src/core.rsindex 69278557..cd9eda36 100644--- a/plugins/optimus/companion/src/core.rs+++ b/plugins/optimus/companion/src/core.rs@@ -355,6 +355,19 @@ fn on_mercury(shared: &Arc<Shared>, msg: &Value) {// Mercury Desktop's voice controls (the notch capsule and the status bar)."mercury/voice/control" => {let action = p.get("action").and_then(Value::as_str).unwrap_or("");+ // Start a session from the app, as if the talk key were pressed (only when none is running).+ if action == "start" {+ if shared.session.lock().unwrap().is_none() {+ let sh = shared.clone();+ shared.rt.spawn(async move {+ if let Err(e) = crate::voice::press(sh, false).await {+ log(format!("start from Mercury: {e}"));+ error_sound();+ }+ });+ }+ return;+ }let live = shared.session.lock().unwrap();if let Some(s) = live.as_ref() {match action {diff --git a/plugins/optimus/companion/src/voice.rs b/plugins/optimus/companion/src/voice.rsindex c1a04068..d6dc48c9 100644--- a/plugins/optimus/companion/src/voice.rs+++ b/plugins/optimus/companion/src/voice.rs@@ -256,6 +256,11 @@ async fn bridge(let mut tock = false;tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);let mut last_narrated = Instant::now() - Duration::from_secs(60);+ // Voice levels for Mercury Desktop's call screen (~12 a second while anyone is audible).+ let mut level_tick = tokio::time::interval(Duration::from_millis(80));+ level_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ let mut mic_peak: i16 = 0;+ let mut levels_quiet = false;// Silence ends the session by itself, so an open mic never streams (and bills) for nothing.let idle_stop = settings().get("idleStopSeconds").and_then(Value::as_f64).unwrap_or(60.0).max(0.0);@@ -370,10 +375,22 @@ async fn bridge(}}},+ _ = level_tick.tick() => {+ let mic_level = if muted.load(std::sync::atomic::Ordering::Relaxed) { 0.0 } else { (mic_peak as f32 / 32768.0 * 3.0).min(1.0) };+ let out_level = (out.take_level() * 2.2).min(1.0);+ mic_peak = 0;+ let quiet = mic_level < 0.02 && out_level < 0.02;+ // Say "silent" once, then stay quiet until someone is audible again.+ if !(quiet && levels_quiet) {+ sh.out.notify("mercury/voice/level", json!({ "callId": call_id, "mic": (mic_level * 100.0).round() / 100.0, "out": (out_level * 100.0).round() / 100.0 }));+ }+ levels_quiet = quiet;+ }frame = mic.recv() => {let Some(frame) = frame else { break Err("the microphone stopped".to_string()) };// Muted: nothing the owner says is sent anywhere.if muted.load(std::sync::atomic::Ordering::Relaxed) { continue; }+ mic_peak = mic_peak.max(frame.iter().map(|s| s.saturating_abs()).max().unwrap_or(0));// With speakers, the mic would hear Mercury: send silence while it talks.let muted = !barge_in && out.recently_audible(350);link.send(LinkCmd::Audio(if muted { silence.clone() } else { frame }));@@ -485,11 +502,16 @@ async fn bridge(}}LinkEvent::AssistantText(t) => {- if !t.is_empty() { transcript.push(json!({ "role": "mercury", "text": t })); }+ if !t.is_empty() {+ // Live, for the call screen and the call's chat thread.+ sh.out.notify("mercury/voice/transcript", json!({ "callId": call_id, "role": "mercury", "text": t }));+ transcript.push(json!({ "role": "mercury", "text": t }));+ }}LinkEvent::UserText(t) => {if t.is_empty() { continue; }last_activity = Instant::now();+ sh.out.notify("mercury/voice/transcript", json!({ "callId": call_id, "role": "owner", "text": t.clone() }));transcript.push(json!({ "role": "owner", "text": t.clone() }));user_said.push((Instant::now(), t.clone()));if user_said.len() > 20 { user_said.remove(0); }