Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat(optimus): status bar shows when Optimus is listening or working; silence ends a session (0.3.1)

Salman Qureshi2f91a67426d1
diff --git a/plugins/optimus/companion/Cargo.toml b/plugins/optimus/companion/Cargo.toml
index a9bcab45..b3d97487 100644
--- a/plugins/optimus/companion/Cargo.toml
+++ b/plugins/optimus/companion/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "mercury-optimus"
-version = "0.3.0"
+version = "0.3.1"
edition = "2024"
rust-version = "1.94"
description = "Optimus: full computer control for Mercury (screen, mouse, keyboard, apps, voice)"
diff --git a/plugins/optimus/companion/src/core.rs b/plugins/optimus/companion/src/core.rs
index e8702b65..39b68cd1 100644
--- a/plugins/optimus/companion/src/core.rs
+++ b/plugins/optimus/companion/src/core.rs
@@ -97,6 +97,21 @@ pub struct Shared {
pub session: Mutex<Option<crate::voice::SessionCtl>>,
/// How the talk key reads, e.g. "Control+Option+M" (empty when there's none).
pub hotkey_label: Mutex<String>,
+ /// Screen operations in progress, and the (listening, working) state last reported.
+ working: Mutex<(u32, Option<(bool, bool)>)>,
+}
+
+impl Shared {
+ /// Tell Mercury whether Optimus is listening and/or working (the TUI status bar shows it).
+ pub fn publish_activity(&self) {
+ let listening = self.session.lock().unwrap().is_some();
+ let mut w = self.working.lock().unwrap();
+ let now = (listening, w.0 > 0);
+ if w.1 != Some(now) {
+ w.1 = Some(now);
+ self.out.notify("mercury/channel/activity", json!({ "listening": now.0, "working": now.1 }));
+ }
+ }
}
/// Run the core until the plugin side goes away.
@@ -111,7 +126,9 @@ pub fn run(inbox: std::sync::mpsc::Receiver<Value>, send: impl Fn(Value) + Send
call_approvals: Mutex::new(HashMap::new()),
session: Mutex::new(None),
hotkey_label: Mutex::new(String::new()),
+ working: Mutex::new((0, None)),
});
+ shared.publish_activity();
start_hotkey(&shared);
@@ -137,9 +154,26 @@ pub fn run(inbox: std::sync::mpsc::Receiver<Value>, send: impl Fn(Value) + Send
s.narrate(line);
}
}
+ let on_screen = matches!(name.as_str(), "look" | "act" | "open" | "windows");
+ if on_screen {
+ shared.working.lock().unwrap().0 += 1;
+ shared.publish_activity();
+ }
rt.spawn_blocking(move || {
let r = sh.ops.lock().unwrap_or_else(|e| e.into_inner()).run(&name, &args);
sh.out.op_result(id, r);
+ if on_screen {
+ {
+ let mut w = sh.working.lock().unwrap();
+ w.0 = w.0.saturating_sub(1);
+ }
+ // Linger a moment: a task's steps come a few seconds apart, and the bar shouldn't flicker.
+ let sh2 = sh.clone();
+ sh.rt.spawn(async move {
+ tokio::time::sleep(Duration::from_secs(4)).await;
+ sh2.publish_activity();
+ });
+ }
});
continue;
}
diff --git a/plugins/optimus/companion/src/voice.rs b/plugins/optimus/companion/src/voice.rs
index 37e1a0f4..e234d6d0 100644
--- a/plugins/optimus/companion/src/voice.rs
+++ b/plugins/optimus/companion/src/voice.rs
@@ -96,10 +96,12 @@ pub async fn press(sh: Arc<Shared>, from_tool: bool) -> Result<String, String> {
}
};
log(format!("listening ({})", audio.description));
+ sh.publish_activity();
let sh2 = sh.clone();
tokio::spawn(async move {
run_session(sh2.clone(), audio, rx, call_id, instructions).await;
*sh2.session.lock().unwrap() = None;
+ sh2.publish_activity();
});
let key = sh.hotkey_label.lock().unwrap().clone();
Ok(if key.is_empty() {
@@ -211,6 +213,9 @@ async fn bridge(
let mut narration: Option<String> = None;
let mut last_narrated = Instant::now() - Duration::from_secs(60);
+ // Silence ends the session by itself, so an open mic never streams (and bills) for nothing.
+ let idle_stop = settings().get("idleStopSeconds").and_then(Value::as_f64).unwrap_or(60.0).max(0.0);
+ let mut last_activity = Instant::now();
let max_minutes = settings().get("maxSessionMinutes").and_then(Value::as_f64).unwrap_or(30.0).clamp(1.0, 240.0);
let deadline = tokio::time::sleep(Duration::from_secs_f64(max_minutes * 60.0));
tokio::pin!(deadline);
@@ -282,7 +287,17 @@ async fn bridge(
deliver(&link, brain, &mut ready_answers, &mut responding);
continue;
}
- if responding || out.busy() { continue; }
+ if responding || out.busy() { last_activity = Instant::now(); continue; }
+ if idle_stop > 0.0
+ && last_activity.elapsed().as_secs_f64() >= idle_stop
+ && last_narrated.elapsed().as_secs_f64() >= idle_stop
+ && pending.lock().unwrap().is_empty()
+ && ready_answers.is_empty()
+ && awaiting.is_none()
+ {
+ log(format!("{call_id}: {idle_stop}s of silence, stopping"));
+ break Ok(());
+ }
if awaiting.is_none() {
let next = sh.call_approvals.lock().unwrap().get_mut(call_id).and_then(|l| if l.is_empty() { None } else { Some(l.remove(0)) });
if let Some((id, prompt)) = next {
@@ -348,6 +363,7 @@ async fn bridge(
}
LinkEvent::Audio(pcm) => out.play(&pcm),
LinkEvent::SpeechStarted => {
+ last_activity = Instant::now();
if barge_in || !out.recently_audible(350) {
out.clear();
link.send(LinkCmd::Cancel);
@@ -358,6 +374,7 @@ async fn bridge(
}
LinkEvent::UserText(t) => {
if t.is_empty() { continue; }
+ last_activity = Instant::now();
transcript.push(json!({ "role": "owner", "text": t.clone() }));
user_said.push((Instant::now(), t.clone()));
if user_said.len() > 20 { user_said.remove(0); }
diff --git a/plugins/optimus/mercury-plugin.template.json b/plugins/optimus/mercury-plugin.template.json
index 26b2b349..1b0f411d 100644
--- a/plugins/optimus/mercury-plugin.template.json
+++ b/plugins/optimus/mercury-plugin.template.json
@@ -2,7 +2,7 @@
"manifestVersion": 1,
"id": "optimus",
"name": "Optimus",
- "version": "0.3.0",
+ "version": "0.3.1",
"description": "Mercury uses your computer like a person: sees the screen, moves the mouse, types and opens apps, by chat or out loud with a talk key. macOS, Windows and Linux. Asks before anything that submits, sends, pays or deletes.",
"kind": "out-of-process",
"author": "Cosmic Stack",
@@ -177,6 +177,19 @@
]
}
},
+ "idleStopSeconds": {
+ "type": "number",
+ "default": 60,
+ "label": "Stop listening after this much silence (seconds)",
+ "description": "The mic closes by itself when you haven't spoken for this long and Mercury isn't working on something, so it never streams (and costs) for nothing. 0 keeps listening until you press the key or say bye.",
+ "group": "voice",
+ "when": {
+ "mode": [
+ "voice",
+ "both"
+ ]
+ }
+ },
"voiceEngine": {
"type": "enum",
"label": "Who listens and speaks?",
diff --git a/src/channels/cli.ts b/src/channels/cli.ts
index 90045b1d..8f7407a5 100644
--- a/src/channels/cli.ts
+++ b/src/channels/cli.ts
@@ -461,6 +461,8 @@ export interface TuiState {
/** A newer Mercury version the background check found — shown as a tiny
* bottom-bar indicator (⬆ vX) in Mercury Code until ignored. */
updateAvailable: string | null;
+ /** What a desktop plugin (Optimus) is doing right now, e.g. "🎙 Optimus listening"; null when idle. */
+ desktopActivity: string | null;
/** Active Mercury Bot chat — transcript is swapped to the bot's thread. */
botChat: { botId: string; botName: string } | null;
/** Per-bot real-time activity (what each bot is doing RIGHT NOW) — rendered
@@ -505,6 +507,7 @@ const defaultState: TuiState = {
statusVerbs: null,
tuiFrozen: false,
updateAvailable: null,
+ desktopActivity: null,
botChat: null,
botLiveActivity: {},
botStreamTails: {},
@@ -2159,6 +2162,12 @@ export class CLIChannel extends BaseChannel {
}
}
+ /** Status-bar note for a desktop plugin's activity (Optimus listening / working); null clears it. */
+ setDesktopActivity(text: string | null): void {
+ if (this.state.desktopActivity === text) return;
+ this.update({ desktopActivity: text });
+ }
+
setSkills(skills: SkillInfo[]): void {
this.update({ skills });
}
diff --git a/src/index.ts b/src/index.ts
index 6509f8cf..ddbe08f2 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -54,7 +54,7 @@ import { buildConversationHistoryPayload, CloudSessionSynchronizer, SessionRepos
import { UserMemoryStore } from './memory/user-memory.js';
import { BotManager } from './bots/bot-manager.js';
import { getPluginManager, peekPluginManager, setProviderLookup } from './plugins/manager.js';
-import { setCallLearning, setChannelHost, setVoiceBriefing } from './plugins/channel-bridge.js';
+import { setCallLearning, setChannelHost, setDesktopActivityHook, setVoiceBriefing } from './plugins/channel-bridge.js';
import { registerPluginsCli } from './cli/plugins-cli.js';
import { SECOND_BRAIN_UNAVAILABLE_MESSAGE } from './memory/second-brain-db.js';
import { isSqliteAvailable } from './utils/sqlite-driver.js';
@@ -2694,6 +2694,10 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {
// Calls know what chats know (identity, about, the owner's profile) and learn like chats do.
setVoiceBriefing(() => agent.voiceBriefing());
setCallLearning((ownerSaid, mercurySaid) => agent.learnFromConversation(ownerSaid, mercurySaid));
+ setDesktopActivityHook((label) => {
+ const cli = channels.get('cli');
+ if (cli instanceof CLIChannel) cli.setDesktopActivity(label);
+ });
pluginManager.setRunsPlugins(true);
pluginManager.setNotifier((m) => agent.notifyAllChannels(m));
void pluginManager.boot().catch((err) => logger.error({ err }, 'Plugin boot failed'));
diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.ts
index 46a8229a..153a0d45 100644
--- a/src/plugins/channel-bridge.ts
+++ b/src/plugins/channel-bridge.ts
@@ -250,6 +250,7 @@ export class PluginChannel extends BaseChannel {
}
detach(): void {
+ if (this.desktop) desktopActivityHook?.(null);
if (live.get(this.manifest.id) === this) live.delete(this.manifest.id);
host?.unregister(this.type, this);
this.ready = false;
@@ -269,6 +270,7 @@ export class PluginChannel extends BaseChannel {
if (method === 'mercury/channel/status') this.onStatusNotification(params);
else if (method === 'mercury/channel/message') this.onMessageNotification(params as IncomingParams);
else if (method === 'mercury/call/ended') this.onCallEnded(params);
+ else if (method === 'mercury/channel/activity') this.onActivity(params);
}
async handleRequest(method: string, params: unknown): Promise<unknown> {
@@ -376,6 +378,12 @@ export class PluginChannel extends BaseChannel {
this.overflow.delete(target);
}
+ /** Plugin API 1.2: what a desktop plugin is doing right now, for the status bar. */
+ private onActivity(params: unknown): void {
+ const p = (params ?? {}) as Record<string, unknown>;
+ desktopActivityHook?.(activityLabel(this.manifest.name, p.listening === true, p.working === true));
+ }
+
private onCallEnded(params: unknown): void {
const p = (params ?? {}) as Record<string, unknown>;
// Learn from the call like a chat (only at Extended: General keeps memory off).
@@ -798,6 +806,20 @@ export function setVoiceBriefing(fn: (() => string) | null): void {
}
/** After a call: let Mercury learn from it like any other conversation. */
+let desktopActivityHook: ((label: string | null) => void) | null = null;
+/** Where desktop activity is shown (the TUI status bar). */
+export function setDesktopActivityHook(fn: ((label: string | null) => void) | null): void {
+ desktopActivityHook = fn;
+}
+
+/** "🎙 Optimus listening", "🖱 Optimus working", both, or null when idle. */
+export function activityLabel(name: string, listening: boolean, working: boolean): string | null {
+ if (listening && working) return `🎙🖱 ${name} working`;
+ if (working) return `🖱 ${name} working`;
+ if (listening) return `🎙 ${name} listening`;
+ return null;
+}
+
let learnFromCall: ((ownerSaid: string, mercurySaid: string) => void) | null = null;
export function setCallLearning(fn: ((ownerSaid: string, mercurySaid: string) => void) | null): void {
learnFromCall = fn;
diff --git a/src/plugins/channel.test.ts b/src/plugins/channel.test.ts
index 15a2f08b..b469b5ae 100644
--- a/src/plugins/channel.test.ts
+++ b/src/plugins/channel.test.ts
@@ -7,7 +7,7 @@ import type { Channel } from '../channels/base.js';
import type { ChannelMessage } from '../types/channel.js';
import { PluginManager, setProviderLookup } from './manager.js';
import { PluginChannel } from './channel-bridge.js';
-import { getPluginChannel, setCallLearning, setChannelHost, setVoiceBriefing, splitMessage, spokenPart, toWhatsAppText, voiceBriefing } from './channel-bridge.js';
+import { activityLabel, getPluginChannel, setCallLearning, setChannelHost, setDesktopActivityHook, setVoiceBriefing, splitMessage, spokenPart, toWhatsAppText, voiceBriefing } from './channel-bridge.js';
import { getPluginTools, resetToolBridge } from './tool-bridge.js';
import { pluginDataDir } from './store.js';
import { manifestSchema } from './types.js';
@@ -542,6 +542,24 @@ describe('plugin channels (plugin API 1.1)', () => {
expect(lines('out.jsonl').filter((m) => m.method === 'mercury/channel/send')).toEqual([]);
});
+ it('reports what Optimus is doing to the status bar, and clears it when it stops', async () => {
+ const labels: Array<string | null> = [];
+ setDesktopActivityHook((l) => labels.push(l));
+ try {
+ const ch = await installDesktop();
+ ch.handleNotification('mercury/channel/activity', { listening: true, working: false });
+ ch.handleNotification('mercury/channel/activity', { listening: true, working: true });
+ ch.handleNotification('mercury/channel/activity', { listening: false, working: false });
+ expect(labels).toEqual(['🎙 Optimus listening', '🎙🖱 Optimus working', null]);
+ await mgr.disable('optimus');
+ await waitFor(() => labels.length === 4);
+ expect(labels[3]).toBeNull();
+ } finally {
+ setDesktopActivityHook(null);
+ }
+ expect(activityLabel('Optimus', false, true)).toBe('🖱 Optimus working');
+ });
+
it('approvals are asked out loud only', async () => {
const ch = await installDesktop();
const answer = ch.askPermission('Submit the job application', 'desktop:call:d3');
diff --git a/src/ui/App.tsx b/src/ui/App.tsx
index 2d460a1f..d53daad5 100644
--- a/src/ui/App.tsx
+++ b/src/ui/App.tsx
@@ -880,7 +880,7 @@ function HeaderBanner({ cols }: { cols: number }): React.ReactNode {
}
function TokenBarView({ state, cols }: { state: TuiState; cols: number }) {
- if (!state.tokenInfo && !state.provider && !state.currentSession) return null;
+ if (!state.tokenInfo && !state.provider && !state.currentSession && !state.desktopActivity) return null;
const saverActive = !!(state.saverInfo && state.saverInfo.state !== 'off');
const saverColor = state.saverInfo?.state === 'auto' ? 'yellow' : 'green';
@@ -911,6 +911,11 @@ function TokenBarView({ state, cols }: { state: TuiState; cols: number }) {
segments.push({ text: `${state.provider.name} · ${state.provider.model}`, color: 'magenta' });
}
+ // 2b. Optimus (desktop plugin) listening / working right now
+ if (state.desktopActivity) {
+ segments.push({ text: state.desktopActivity, color: 'yellow' });
+ }
+
// 3. Session
if (state.currentSession) {
segments.push({ text: `${state.currentSession.alias} [${state.currentSession.shortId}]`, color: 'cyan' });
@@ -2718,6 +2723,7 @@ export function MercuryCodeView({
// '' = not read yet (async refresh pending); hide the segment until then.
const hasGit = git.branch !== '' && git.branch !== 'no-git' && git.branch !== 'not-a-git-repo';
const rightSegs: Array<{ text: string; color: string }> = [{ text: mc.dirName, color: THEME_ACCENT }];
+ if (state.desktopActivity) rightSegs.push({ text: state.desktopActivity, color: 'yellow' });
if (hasGit) {
const gitBits = [`⎇ ${git.branch}`];
if (git.ahead > 0) gitBits.push(`↑${git.ahead}`);