Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat(optimus): status bar shows when Optimus is listening or working; silence ends a session (0.3.1)
Salman Qureshi2f91a67426d1
diff --git a/plugins/optimus/companion/Cargo.toml b/plugins/optimus/companion/Cargo.tomlindex a9bcab45..b3d97487 100644--- a/plugins/optimus/companion/Cargo.toml+++ b/plugins/optimus/companion/Cargo.toml@@ -1,6 +1,6 @@[package]name = "mercury-optimus"-version = "0.3.0"+version = "0.3.1"edition = "2024"rust-version = "1.94"description = "Optimus: full computer control for Mercury (screen, mouse, keyboard, apps, voice)"diff --git a/plugins/optimus/companion/src/core.rs b/plugins/optimus/companion/src/core.rsindex e8702b65..39b68cd1 100644--- a/plugins/optimus/companion/src/core.rs+++ b/plugins/optimus/companion/src/core.rs@@ -97,6 +97,21 @@ pub struct Shared {pub session: Mutex<Option<crate::voice::SessionCtl>>,/// How the talk key reads, e.g. "Control+Option+M" (empty when there's none).pub hotkey_label: Mutex<String>,+ /// Screen operations in progress, and the (listening, working) state last reported.+ working: Mutex<(u32, Option<(bool, bool)>)>,+}++impl Shared {+ /// Tell Mercury whether Optimus is listening and/or working (the TUI status bar shows it).+ pub fn publish_activity(&self) {+ let listening = self.session.lock().unwrap().is_some();+ let mut w = self.working.lock().unwrap();+ let now = (listening, w.0 > 0);+ if w.1 != Some(now) {+ w.1 = Some(now);+ self.out.notify("mercury/channel/activity", json!({ "listening": now.0, "working": now.1 }));+ }+ }}/// Run the core until the plugin side goes away.@@ -111,7 +126,9 @@ pub fn run(inbox: std::sync::mpsc::Receiver<Value>, send: impl Fn(Value) + Sendcall_approvals: Mutex::new(HashMap::new()),session: Mutex::new(None),hotkey_label: Mutex::new(String::new()),+ working: Mutex::new((0, None)),});+ shared.publish_activity();start_hotkey(&shared);@@ -137,9 +154,26 @@ pub fn run(inbox: std::sync::mpsc::Receiver<Value>, send: impl Fn(Value) + Sends.narrate(line);}}+ let on_screen = matches!(name.as_str(), "look" | "act" | "open" | "windows");+ if on_screen {+ shared.working.lock().unwrap().0 += 1;+ shared.publish_activity();+ }rt.spawn_blocking(move || {let r = sh.ops.lock().unwrap_or_else(|e| e.into_inner()).run(&name, &args);sh.out.op_result(id, r);+ if on_screen {+ {+ let mut w = sh.working.lock().unwrap();+ w.0 = w.0.saturating_sub(1);+ }+ // Linger a moment: a task's steps come a few seconds apart, and the bar shouldn't flicker.+ let sh2 = sh.clone();+ sh.rt.spawn(async move {+ tokio::time::sleep(Duration::from_secs(4)).await;+ sh2.publish_activity();+ });+ }});continue;}diff --git a/plugins/optimus/companion/src/voice.rs b/plugins/optimus/companion/src/voice.rsindex 37e1a0f4..e234d6d0 100644--- a/plugins/optimus/companion/src/voice.rs+++ b/plugins/optimus/companion/src/voice.rs@@ -96,10 +96,12 @@ pub async fn press(sh: Arc<Shared>, from_tool: bool) -> Result<String, String> {}};log(format!("listening ({})", audio.description));+ sh.publish_activity();let sh2 = sh.clone();tokio::spawn(async move {run_session(sh2.clone(), audio, rx, call_id, instructions).await;*sh2.session.lock().unwrap() = None;+ sh2.publish_activity();});let key = sh.hotkey_label.lock().unwrap().clone();Ok(if key.is_empty() {@@ -211,6 +213,9 @@ async fn bridge(let mut narration: Option<String> = None;let mut last_narrated = Instant::now() - Duration::from_secs(60);+ // Silence ends the session by itself, so an open mic never streams (and bills) for nothing.+ let idle_stop = settings().get("idleStopSeconds").and_then(Value::as_f64).unwrap_or(60.0).max(0.0);+ let mut last_activity = Instant::now();let max_minutes = settings().get("maxSessionMinutes").and_then(Value::as_f64).unwrap_or(30.0).clamp(1.0, 240.0);let deadline = tokio::time::sleep(Duration::from_secs_f64(max_minutes * 60.0));tokio::pin!(deadline);@@ -282,7 +287,17 @@ async fn bridge(deliver(&link, brain, &mut ready_answers, &mut responding);continue;}- if responding || out.busy() { continue; }+ if responding || out.busy() { last_activity = Instant::now(); continue; }+ if idle_stop > 0.0+ && last_activity.elapsed().as_secs_f64() >= idle_stop+ && last_narrated.elapsed().as_secs_f64() >= idle_stop+ && pending.lock().unwrap().is_empty()+ && ready_answers.is_empty()+ && awaiting.is_none()+ {+ log(format!("{call_id}: {idle_stop}s of silence, stopping"));+ break Ok(());+ }if awaiting.is_none() {let next = sh.call_approvals.lock().unwrap().get_mut(call_id).and_then(|l| if l.is_empty() { None } else { Some(l.remove(0)) });if let Some((id, prompt)) = next {@@ -348,6 +363,7 @@ async fn bridge(}LinkEvent::Audio(pcm) => out.play(&pcm),LinkEvent::SpeechStarted => {+ last_activity = Instant::now();if barge_in || !out.recently_audible(350) {out.clear();link.send(LinkCmd::Cancel);@@ -358,6 +374,7 @@ async fn bridge(}LinkEvent::UserText(t) => {if t.is_empty() { continue; }+ last_activity = Instant::now();transcript.push(json!({ "role": "owner", "text": t.clone() }));user_said.push((Instant::now(), t.clone()));if user_said.len() > 20 { user_said.remove(0); }diff --git a/plugins/optimus/mercury-plugin.template.json b/plugins/optimus/mercury-plugin.template.jsonindex 26b2b349..1b0f411d 100644--- a/plugins/optimus/mercury-plugin.template.json+++ b/plugins/optimus/mercury-plugin.template.json@@ -2,7 +2,7 @@"manifestVersion": 1,"id": "optimus","name": "Optimus",- "version": "0.3.0",+ "version": "0.3.1","description": "Mercury uses your computer like a person: sees the screen, moves the mouse, types and opens apps, by chat or out loud with a talk key. macOS, Windows and Linux. Asks before anything that submits, sends, pays or deletes.","kind": "out-of-process","author": "Cosmic Stack",@@ -177,6 +177,19 @@]}},+ "idleStopSeconds": {+ "type": "number",+ "default": 60,+ "label": "Stop listening after this much silence (seconds)",+ "description": "The mic closes by itself when you haven't spoken for this long and Mercury isn't working on something, so it never streams (and costs) for nothing. 0 keeps listening until you press the key or say bye.",+ "group": "voice",+ "when": {+ "mode": [+ "voice",+ "both"+ ]+ }+ },"voiceEngine": {"type": "enum","label": "Who listens and speaks?",diff --git a/src/channels/cli.ts b/src/channels/cli.tsindex 90045b1d..8f7407a5 100644--- a/src/channels/cli.ts+++ b/src/channels/cli.ts@@ -461,6 +461,8 @@ export interface TuiState {/** A newer Mercury version the background check found — shown as a tiny* bottom-bar indicator (⬆ vX) in Mercury Code until ignored. */updateAvailable: string | null;+ /** What a desktop plugin (Optimus) is doing right now, e.g. "🎙 Optimus listening"; null when idle. */+ desktopActivity: string | null;/** Active Mercury Bot chat — transcript is swapped to the bot's thread. */botChat: { botId: string; botName: string } | null;/** Per-bot real-time activity (what each bot is doing RIGHT NOW) — rendered@@ -505,6 +507,7 @@ const defaultState: TuiState = {statusVerbs: null,tuiFrozen: false,updateAvailable: null,+ desktopActivity: null,botChat: null,botLiveActivity: {},botStreamTails: {},@@ -2159,6 +2162,12 @@ export class CLIChannel extends BaseChannel {}}+ /** Status-bar note for a desktop plugin's activity (Optimus listening / working); null clears it. */+ setDesktopActivity(text: string | null): void {+ if (this.state.desktopActivity === text) return;+ this.update({ desktopActivity: text });+ }+setSkills(skills: SkillInfo[]): void {this.update({ skills });}diff --git a/src/index.ts b/src/index.tsindex 6509f8cf..ddbe08f2 100644--- a/src/index.ts+++ b/src/index.ts@@ -54,7 +54,7 @@ import { buildConversationHistoryPayload, CloudSessionSynchronizer, SessionReposimport { UserMemoryStore } from './memory/user-memory.js';import { BotManager } from './bots/bot-manager.js';import { getPluginManager, peekPluginManager, setProviderLookup } from './plugins/manager.js';-import { setCallLearning, setChannelHost, setVoiceBriefing } from './plugins/channel-bridge.js';+import { setCallLearning, setChannelHost, setDesktopActivityHook, setVoiceBriefing } from './plugins/channel-bridge.js';import { registerPluginsCli } from './cli/plugins-cli.js';import { SECOND_BRAIN_UNAVAILABLE_MESSAGE } from './memory/second-brain-db.js';import { isSqliteAvailable } from './utils/sqlite-driver.js';@@ -2694,6 +2694,10 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {// Calls know what chats know (identity, about, the owner's profile) and learn like chats do.setVoiceBriefing(() => agent.voiceBriefing());setCallLearning((ownerSaid, mercurySaid) => agent.learnFromConversation(ownerSaid, mercurySaid));+ setDesktopActivityHook((label) => {+ const cli = channels.get('cli');+ if (cli instanceof CLIChannel) cli.setDesktopActivity(label);+ });pluginManager.setRunsPlugins(true);pluginManager.setNotifier((m) => agent.notifyAllChannels(m));void pluginManager.boot().catch((err) => logger.error({ err }, 'Plugin boot failed'));diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.tsindex 46a8229a..153a0d45 100644--- a/src/plugins/channel-bridge.ts+++ b/src/plugins/channel-bridge.ts@@ -250,6 +250,7 @@ export class PluginChannel extends BaseChannel {}detach(): void {+ if (this.desktop) desktopActivityHook?.(null);if (live.get(this.manifest.id) === this) live.delete(this.manifest.id);host?.unregister(this.type, this);this.ready = false;@@ -269,6 +270,7 @@ export class PluginChannel extends BaseChannel {if (method === 'mercury/channel/status') this.onStatusNotification(params);else if (method === 'mercury/channel/message') this.onMessageNotification(params as IncomingParams);else if (method === 'mercury/call/ended') this.onCallEnded(params);+ else if (method === 'mercury/channel/activity') this.onActivity(params);}async handleRequest(method: string, params: unknown): Promise<unknown> {@@ -376,6 +378,12 @@ export class PluginChannel extends BaseChannel {this.overflow.delete(target);}+ /** Plugin API 1.2: what a desktop plugin is doing right now, for the status bar. */+ private onActivity(params: unknown): void {+ const p = (params ?? {}) as Record<string, unknown>;+ desktopActivityHook?.(activityLabel(this.manifest.name, p.listening === true, p.working === true));+ }+private onCallEnded(params: unknown): void {const p = (params ?? {}) as Record<string, unknown>;// Learn from the call like a chat (only at Extended: General keeps memory off).@@ -798,6 +806,20 @@ export function setVoiceBriefing(fn: (() => string) | null): void {}/** After a call: let Mercury learn from it like any other conversation. */+let desktopActivityHook: ((label: string | null) => void) | null = null;+/** Where desktop activity is shown (the TUI status bar). */+export function setDesktopActivityHook(fn: ((label: string | null) => void) | null): void {+ desktopActivityHook = fn;+}++/** "🎙 Optimus listening", "🖱 Optimus working", both, or null when idle. */+export function activityLabel(name: string, listening: boolean, working: boolean): string | null {+ if (listening && working) return `🎙🖱 ${name} working`;+ if (working) return `🖱 ${name} working`;+ if (listening) return `🎙 ${name} listening`;+ return null;+}+let learnFromCall: ((ownerSaid: string, mercurySaid: string) => void) | null = null;export function setCallLearning(fn: ((ownerSaid: string, mercurySaid: string) => void) | null): void {learnFromCall = fn;diff --git a/src/plugins/channel.test.ts b/src/plugins/channel.test.tsindex 15a2f08b..b469b5ae 100644--- a/src/plugins/channel.test.ts+++ b/src/plugins/channel.test.ts@@ -7,7 +7,7 @@ import type { Channel } from '../channels/base.js';import type { ChannelMessage } from '../types/channel.js';import { PluginManager, setProviderLookup } from './manager.js';import { PluginChannel } from './channel-bridge.js';-import { getPluginChannel, setCallLearning, setChannelHost, setVoiceBriefing, splitMessage, spokenPart, toWhatsAppText, voiceBriefing } from './channel-bridge.js';+import { activityLabel, getPluginChannel, setCallLearning, setChannelHost, setDesktopActivityHook, setVoiceBriefing, splitMessage, spokenPart, toWhatsAppText, voiceBriefing } from './channel-bridge.js';import { getPluginTools, resetToolBridge } from './tool-bridge.js';import { pluginDataDir } from './store.js';import { manifestSchema } from './types.js';@@ -542,6 +542,24 @@ describe('plugin channels (plugin API 1.1)', () => {expect(lines('out.jsonl').filter((m) => m.method === 'mercury/channel/send')).toEqual([]);});+ it('reports what Optimus is doing to the status bar, and clears it when it stops', async () => {+ const labels: Array<string | null> = [];+ setDesktopActivityHook((l) => labels.push(l));+ try {+ const ch = await installDesktop();+ ch.handleNotification('mercury/channel/activity', { listening: true, working: false });+ ch.handleNotification('mercury/channel/activity', { listening: true, working: true });+ ch.handleNotification('mercury/channel/activity', { listening: false, working: false });+ expect(labels).toEqual(['🎙 Optimus listening', '🎙🖱 Optimus working', null]);+ await mgr.disable('optimus');+ await waitFor(() => labels.length === 4);+ expect(labels[3]).toBeNull();+ } finally {+ setDesktopActivityHook(null);+ }+ expect(activityLabel('Optimus', false, true)).toBe('🖱 Optimus working');+ });+it('approvals are asked out loud only', async () => {const ch = await installDesktop();const answer = ch.askPermission('Submit the job application', 'desktop:call:d3');diff --git a/src/ui/App.tsx b/src/ui/App.tsxindex 2d460a1f..d53daad5 100644--- a/src/ui/App.tsx+++ b/src/ui/App.tsx@@ -880,7 +880,7 @@ function HeaderBanner({ cols }: { cols: number }): React.ReactNode {}function TokenBarView({ state, cols }: { state: TuiState; cols: number }) {- if (!state.tokenInfo && !state.provider && !state.currentSession) return null;+ if (!state.tokenInfo && !state.provider && !state.currentSession && !state.desktopActivity) return null;const saverActive = !!(state.saverInfo && state.saverInfo.state !== 'off');const saverColor = state.saverInfo?.state === 'auto' ? 'yellow' : 'green';@@ -911,6 +911,11 @@ function TokenBarView({ state, cols }: { state: TuiState; cols: number }) {segments.push({ text: `${state.provider.name} · ${state.provider.model}`, color: 'magenta' });}+ // 2b. Optimus (desktop plugin) listening / working right now+ if (state.desktopActivity) {+ segments.push({ text: state.desktopActivity, color: 'yellow' });+ }+// 3. Sessionif (state.currentSession) {segments.push({ text: `${state.currentSession.alias} [${state.currentSession.shortId}]`, color: 'cyan' });@@ -2718,6 +2723,7 @@ export function MercuryCodeView({// '' = not read yet (async refresh pending); hide the segment until then.const hasGit = git.branch !== '' && git.branch !== 'no-git' && git.branch !== 'not-a-git-repo';const rightSegs: Array<{ text: string; color: string }> = [{ text: mc.dirName, color: THEME_ACCENT }];+ if (state.desktopActivity) rightSegs.push({ text: state.desktopActivity, color: 'yellow' });if (hasGit) {const gitBits = [`⎇ ${git.branch}`];if (git.ahead > 0) gitBits.push(`↑${git.ahead}`);