Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat(optimus): start from Mercury, live transcript, voice levels

Salman Qureshi90646fd54722
diff --git a/plugins/optimus/companion/src/audio.rs b/plugins/optimus/companion/src/audio.rs
index a4d2d00d..c2d295bb 100644
--- a/plugins/optimus/companion/src/audio.rs
+++ b/plugins/optimus/companion/src/audio.rs
@@ -9,7 +9,7 @@
//! Mercury talks.
use std::collections::VecDeque;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
@@ -28,6 +28,8 @@ pub struct Output {
start: Instant,
/// Milliseconds since `start` when the speaker last played sound.
last_sound: AtomicU64,
+ /// Loudest sample played since the last `take_level` (f32 bits), for the app's call circles.
+ peak: AtomicU32,
}
impl Output {
@@ -99,10 +101,19 @@ impl Output {
}
}
+ /// How loud the speaker was since the last call (0-1), then start over.
+ pub fn take_level(&self) -> f32 {
+ f32::from_bits(self.peak.swap(0, Ordering::Relaxed))
+ }
+
fn next(&self) -> f32 {
let s = self.queue.lock().unwrap().pop_front();
match s {
Some(v) => {
+ let a = v.abs();
+ if a > f32::from_bits(self.peak.load(Ordering::Relaxed)) {
+ self.peak.store(a.to_bits(), Ordering::Relaxed);
+ }
// Only real sound (speech, chimes) counts: the quiet thinking tick doesn't mute the mic.
if v.abs() > 0.06 {
self.last_sound.store(self.start.elapsed().as_millis() as u64, Ordering::Relaxed);
@@ -159,6 +170,7 @@ pub fn open() -> Result<Audio, String> {
rate: out_cfg.sample_rate().0,
start: Instant::now(),
last_sound: AtomicU64::new(0),
+ peak: AtomicU32::new(0),
});
let in_stream = build_input(&input, &in_cfg, mic_tx)?;
let out_stream = build_output(&output, &out_cfg, out.clone())?;
diff --git a/plugins/optimus/companion/src/core.rs b/plugins/optimus/companion/src/core.rs
index 69278557..cd9eda36 100644
--- a/plugins/optimus/companion/src/core.rs
+++ b/plugins/optimus/companion/src/core.rs
@@ -355,6 +355,19 @@ fn on_mercury(shared: &Arc<Shared>, msg: &Value) {
// Mercury Desktop's voice controls (the notch capsule and the status bar).
"mercury/voice/control" => {
let action = p.get("action").and_then(Value::as_str).unwrap_or("");
+ // Start a session from the app, as if the talk key were pressed (only when none is running).
+ if action == "start" {
+ if shared.session.lock().unwrap().is_none() {
+ let sh = shared.clone();
+ shared.rt.spawn(async move {
+ if let Err(e) = crate::voice::press(sh, false).await {
+ log(format!("start from Mercury: {e}"));
+ error_sound();
+ }
+ });
+ }
+ return;
+ }
let live = shared.session.lock().unwrap();
if let Some(s) = live.as_ref() {
match action {
diff --git a/plugins/optimus/companion/src/voice.rs b/plugins/optimus/companion/src/voice.rs
index c1a04068..d6dc48c9 100644
--- a/plugins/optimus/companion/src/voice.rs
+++ b/plugins/optimus/companion/src/voice.rs
@@ -256,6 +256,11 @@ async fn bridge(
let mut tock = false;
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut last_narrated = Instant::now() - Duration::from_secs(60);
+ // Voice levels for Mercury Desktop's call screen (~12 a second while anyone is audible).
+ let mut level_tick = tokio::time::interval(Duration::from_millis(80));
+ level_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ let mut mic_peak: i16 = 0;
+ let mut levels_quiet = false;
// Silence ends the session by itself, so an open mic never streams (and bills) for nothing.
let idle_stop = settings().get("idleStopSeconds").and_then(Value::as_f64).unwrap_or(60.0).max(0.0);
@@ -370,10 +375,22 @@ async fn bridge(
}
}
},
+ _ = level_tick.tick() => {
+ let mic_level = if muted.load(std::sync::atomic::Ordering::Relaxed) { 0.0 } else { (mic_peak as f32 / 32768.0 * 3.0).min(1.0) };
+ let out_level = (out.take_level() * 2.2).min(1.0);
+ mic_peak = 0;
+ let quiet = mic_level < 0.02 && out_level < 0.02;
+ // Say "silent" once, then stay quiet until someone is audible again.
+ if !(quiet && levels_quiet) {
+ sh.out.notify("mercury/voice/level", json!({ "callId": call_id, "mic": (mic_level * 100.0).round() / 100.0, "out": (out_level * 100.0).round() / 100.0 }));
+ }
+ levels_quiet = quiet;
+ }
frame = mic.recv() => {
let Some(frame) = frame else { break Err("the microphone stopped".to_string()) };
// Muted: nothing the owner says is sent anywhere.
if muted.load(std::sync::atomic::Ordering::Relaxed) { continue; }
+ mic_peak = mic_peak.max(frame.iter().map(|s| s.saturating_abs()).max().unwrap_or(0));
// With speakers, the mic would hear Mercury: send silence while it talks.
let muted = !barge_in && out.recently_audible(350);
link.send(LinkCmd::Audio(if muted { silence.clone() } else { frame }));
@@ -485,11 +502,16 @@ async fn bridge(
}
}
LinkEvent::AssistantText(t) => {
- if !t.is_empty() { transcript.push(json!({ "role": "mercury", "text": t })); }
+ if !t.is_empty() {
+ // Live, for the call screen and the call's chat thread.
+ sh.out.notify("mercury/voice/transcript", json!({ "callId": call_id, "role": "mercury", "text": t }));
+ transcript.push(json!({ "role": "mercury", "text": t }));
+ }
}
LinkEvent::UserText(t) => {
if t.is_empty() { continue; }
last_activity = Instant::now();
+ sh.out.notify("mercury/voice/transcript", json!({ "callId": call_id, "role": "owner", "text": t.clone() }));
transcript.push(json!({ "role": "owner", "text": t.clone() }));
user_said.push((Instant::now(), t.clone()));
if user_said.len() > 20 { user_said.remove(0); }