Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

Windows and Linux: Wayland dictation and « OK Coucou »

Clauded775e813a11a
diff --git a/windows/src-tauri/src/claude.rs b/windows/src-tauri/src/claude.rs
index a18766c1..7cd9a8e5 100644
--- a/windows/src-tauri/src/claude.rs
+++ b/windows/src-tauri/src/claude.rs
@@ -182,6 +182,21 @@ pub async fn complete(model: &str, system: &str, user: &str, max_tokens: u32, ti
Ok(text)
}
+/// One turn of « OK Coucou »'s conversation with tools: the API's whole answer
+/// (content blocks and stop_reason).
+pub async fn voice_turn(model: &str, system: &str, messages: &[Value], tools: &[Value]) -> Result<Value, String> {
+ let key = secrets::get(KEY).ok_or_else(|| t("API key missing. Open settings."))?;
+ let endpoint = endpoint()?;
+ let body = json!({
+ "model": model,
+ "max_tokens": 1024,
+ "system": system,
+ "messages": messages,
+ "tools": tools,
+ });
+ call_with_timeout(&endpoint, &key, &body, Duration::from_secs(45)).await
+}
+
async fn call(endpoint: &Url, key: &str, body: &Value) -> Result<Value, String> {
call_with_timeout(endpoint, key, body, Duration::from_secs(90)).await
}
diff --git a/windows/src-tauri/src/flow/mod.rs b/windows/src-tauri/src/flow/mod.rs
index 0db2c29c..3f77842f 100644
--- a/windows/src-tauri/src/flow/mod.rs
+++ b/windows/src-tauri/src/flow/mod.rs
@@ -21,6 +21,8 @@ pub mod os;
#[cfg(target_os = "linux")]
#[path = "os_linux.rs"]
pub mod os;
+#[cfg(target_os = "linux")]
+pub mod wayland;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{self, RecvTimeoutError, Sender};
@@ -60,7 +62,8 @@ pub struct Flow {
struct Running {
tx: Sender<(KeyEvent, Instant)>,
- _listener: Option<os::KeyListener>,
+ /// The OS's key listener (or, on Wayland, the portal's shortcut); dropping it stops it.
+ _listener: Option<Box<dyn std::any::Any + Send>>,
}
impl Flow {
@@ -124,9 +127,15 @@ pub fn setup(app: &AppHandle) {
if prefs.enabled {
enable(app, &flow);
}
+ crate::voice::sync(app);
}
+/// The key as the pages write it ("right_ctrl"…), or "wayland" where the
+/// desktop holds the shortcut.
fn key_id(key: logic::FlowKey) -> String {
+ if os::keys_supported() == Err("wayland") {
+ return "wayland".into();
+ }
serde_json::to_value(key).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default()
}
@@ -137,14 +146,16 @@ fn enable(app: &AppHandle, flow: &Arc<Flow>) {
}
let prefs = flow.store.lock().unwrap().prefs.clone();
let (tx, rx) = mpsc::channel::<(KeyEvent, Instant)>();
- let listener = match os::keys_supported() {
+ let listener: Option<Box<dyn std::any::Any + Send>> = match os::keys_supported() {
Ok(()) => match os::listen(prefs.key, tx.clone()) {
- Ok(l) => Some(l),
+ Ok(l) => Some(Box::new(l)),
Err(err) => {
crate::log::line(format!("flow: the key can't be watched: {err}"));
None
}
},
+ #[cfg(target_os = "linux")]
+ Err("wayland") => wayland::listen(tx.clone()).map(|h| Box::new(h) as Box<dyn std::any::Any + Send>),
Err(why) => {
crate::log::line(format!("flow: no key on this desktop ({why}); the pill's buttons still work"));
None
@@ -172,7 +183,7 @@ fn disable(app: &AppHandle, flow: &Arc<Flow>) {
s.enabled = false;
s.phase = "idle".into();
});
- if !crate::notes::is_recording(app) {
+ if !crate::notes::is_recording(app) && !crate::voice::is_listening() {
speech::unload();
}
}
@@ -387,15 +398,7 @@ fn process(app: &AppHandle, flow: &Arc<Flow>, samples: &[f32], front: Option<&(S
os::wait_key_released(key);
let terminal = front.is_some_and(|(id, _)| is_terminal(id));
- let outcome = if os_lacks_paste() {
- if paste::copy(&final_text) {
- paste::Outcome::Copied
- } else {
- paste::Outcome::Failed
- }
- } else {
- paste::paste(&final_text, terminal)
- };
+ let outcome = paste::paste(&final_text, terminal);
let entry = {
let mut store = flow.store.lock().unwrap();
store.record(&final_text, front.map(|(_, n)| n.as_str()), seconds, store::today(), store::now_ms())
@@ -423,19 +426,6 @@ fn process(app: &AppHandle, flow: &Arc<Flow>, samples: &[f32], front: Option<&(S
Ok(())
}
-/// Wayland: XTest reaches only XWayland apps, so the words are copied for the
-/// user to paste instead of being typed into the wrong place.
-fn os_lacks_paste() -> bool {
- #[cfg(target_os = "linux")]
- {
- os::is_wayland()
- }
- #[cfg(windows)]
- {
- false
- }
-}
-
/// "Paste again": the last dictation, in the app in front now.
fn paste_again(app: &AppHandle, flow: &Arc<Flow>) {
let Some(text) = flow.store.lock().unwrap().last_text() else { return };
@@ -508,6 +498,13 @@ pub fn flow_set_prefs(app: AppHandle, flow: State<Arc<Flow>>, prefs: Prefs) {
if prefs.enabled && (!before.enabled || restart) {
enable(&app, &flow);
}
+ if before.voice != prefs.voice || before.mic != prefs.mic {
+ if before.mic != prefs.mic {
+ crate::voice::restart(&app);
+ } else {
+ crate::voice::sync(&app);
+ }
+ }
let _ = app.emit("flow-prefs", &prefs);
}
diff --git a/windows/src-tauri/src/flow/os_linux.rs b/windows/src-tauri/src/flow/os_linux.rs
index f417c359..bad4dfb3 100644
--- a/windows/src-tauri/src/flow/os_linux.rs
+++ b/windows/src-tauri/src/flow/os_linux.rs
@@ -170,8 +170,12 @@ pub fn wait_key_released(_key: FlowKey) {
std::thread::sleep(Duration::from_millis(30));
}
-/// Ctrl+V (Ctrl+Shift+V for a terminal) in the app in front, through XTest.
+/// Ctrl+V (Ctrl+Shift+V for a terminal) in the app in front: through XTest on
+/// X11, through the RemoteDesktop portal on Wayland.
pub fn send_paste(shift: bool) -> Result<(), String> {
+ if is_wayland() {
+ return super::wayland::send_paste(shift);
+ }
let (conn, screen) = x11rb::connect(None).map_err(|e| e.to_string())?;
let root = conn.setup().roots[screen].root;
let keymap = Keymap::read(&conn).ok_or("no keyboard map")?;
@@ -214,6 +218,9 @@ fn class_of(conn: &RustConnection, window: u32) -> Option<(String, String)> {
/// The app in front: (id, name). None on Wayland, where only the desktop knows.
pub fn foreground_app() -> Option<(String, String)> {
+ if is_wayland() {
+ return None;
+ }
let (conn, screen) = x11rb::connect(None).ok()?;
let root = conn.setup().roots[screen].root;
let active = atom(&conn, "_NET_ACTIVE_WINDOW")?;
diff --git a/windows/src-tauri/src/flow/pill.rs b/windows/src-tauri/src/flow/pill.rs
index 6a5c2af1..21de293b 100644
--- a/windows/src-tauri/src/flow/pill.rs
+++ b/windows/src-tauri/src/flow/pill.rs
@@ -39,6 +39,8 @@ pub struct PillState {
pub announcement: Option<Announcement>,
/// Dictation is on (the resting pill shows).
pub enabled: bool,
+ /// « OK Coucou » is talking with you.
+ pub voice: Option<crate::voice::VoiceView>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
@@ -141,6 +143,11 @@ pub fn setup(app: &AppHandle) {
}
}
+/// A dictation is under way: the microphone is Flow's.
+pub fn dictating(app: &AppHandle) -> bool {
+ get(app).is_some_and(|p| matches!(p.state.lock().unwrap().phase.as_str(), "listening" | "locked" | "thinking"))
+}
+
pub fn get(app: &AppHandle) -> Option<Arc<Pill>> {
app.try_state::<Arc<Pill>>().map(|s| s.inner().clone())
}
@@ -175,7 +182,7 @@ pub fn update(app: &AppHandle, change: impl FnOnce(&mut PillState)) {
(s.clone(), s.edge != edge_before)
};
let _ = app.emit_to(LABEL, "pill-state", &state);
- let visible = state.enabled || state.note_started_at.is_some() || state.announcement.is_some() || state.phase != "idle";
+ let visible = state.enabled || state.note_started_at.is_some() || state.announcement.is_some() || state.voice.is_some() || state.phase != "idle";
let was = pill.shown.swap(visible, Ordering::Relaxed);
if visible && (!was || edge_changed) {
place(app, &pill, &state.edge, state.phase == "listening");
diff --git a/windows/src-tauri/src/flow/store.rs b/windows/src-tauri/src/flow/store.rs
index 8bf7696e..99b38f22 100644
--- a/windows/src-tauri/src/flow/store.rs
+++ b/windows/src-tauri/src/flow/store.rs
@@ -101,6 +101,12 @@ pub struct Prefs {
pub summary_engine: SummaryEngine,
/// Open the Coucou window when Coucou is opened (not when it starts with the computer).
pub open_at_launch: bool,
+ /// « OK Coucou »: listen for it and answer with Claude (needs the Claude key).
+ pub voice: bool,
+ /// How Coucou answers out loud: "system", "elevenlabs" or "off".
+ pub voice_output: String,
+ /// The ElevenLabs voice: "female" or "male".
+ pub voice_gender: String,
}
impl Default for Prefs {
@@ -115,6 +121,9 @@ impl Default for Prefs {
default_style: Style::Natural,
summary_engine: SummaryEngine::Claude,
open_at_launch: true,
+ voice: false,
+ voice_output: "system".into(),
+ voice_gender: "female".into(),
}
}
}
diff --git a/windows/src-tauri/src/flow/wayland.rs b/windows/src-tauri/src/flow/wayland.rs
new file mode 100644
index 00000000..26a37f69
--- /dev/null
+++ b/windows/src-tauri/src/flow/wayland.rs
@@ -0,0 +1,203 @@
+// Flow dictation on Wayland, where no app may watch the keyboard or type into
+// another one.
+//
+// The key: a shortcut held through the desktop's GlobalShortcuts portal
+// (portal.rs). The desktop tells when it is pressed (Activated) and released
+// (Deactivated), which is all hold-to-talk needs; the user picks the keys in
+// the desktop's own dialog, Ctrl+Alt+V suggested.
+//
+// The paste: Ctrl+V sent through the RemoteDesktop portal, keyboard only. The
+// desktop asks once whether Coucou may control the keyboard; the answer is
+// remembered with the restore token the portal gives back, kept in Coucou's
+// folder. Where neither works, the words stay on the clipboard and the pill
+// says so.
+
+use std::collections::HashMap;
+use std::sync::mpsc::Sender;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use dbus::arg::{PropMap, RefArg, Variant};
+use dbus::blocking::{BlockingSender, Connection};
+use dbus::message::MatchRule;
+use dbus::Path;
+
+use super::KeyEvent;
+use crate::portal;
+
+pub const SHORTCUT_ID: &str = "flowDictate";
+
+/// The portal session holding the dictation shortcut; dropping it gives the
+/// shortcut back.
+pub struct HoldKey {
+ _handle: portal::Handle,
+}
+
+pub fn listen(tx: Sender<(KeyEvent, Instant)>) -> Option<HoldKey> {
+ let handle = portal::start(move |event| match event {
+ portal::Event::Activated(id) if id == SHORTCUT_ID => {
+ let _ = tx.send((KeyEvent::Down, Instant::now()));
+ }
+ portal::Event::Deactivated(id) if id == SHORTCUT_ID => {
+ let _ = tx.send((KeyEvent::Up, Instant::now()));
+ }
+ portal::Event::Bound { .. } => crate::log::line("flow: dictation shortcut held by the desktop"),
+ portal::Event::Failed { reason, .. } | portal::Event::Unavailable(reason) => {
+ crate::log::line(format!("flow: no dictation shortcut from the desktop ({reason})"));
+ }
+ _ => {}
+ })?;
+ let wanted = portal::Wanted {
+ id: SHORTCUT_ID.into(),
+ description: crate::i18n::t("Hold to dictate"),
+ trigger: Some("CTRL+ALT+v".into()),
+ };
+ if !handle.bind(1, vec![wanted]) {
+ return None;
+ }
+ Some(HoldKey { _handle: handle })
+}
+
+// ── The paste ─────────────────────────────────────────────────────────────────
+
+const DEST: &str = "org.freedesktop.portal.Desktop";
+const OBJECT: &str = "/org/freedesktop/portal/desktop";
+const IFACE: &str = "org.freedesktop.portal.RemoteDesktop";
+/// Keysyms of Ctrl, Shift and v.
+const XK_CONTROL_L: i32 = 0xffe3;
+const XK_SHIFT_L: i32 = 0xffe1;
+const XK_V: i32 = 0x0076;
+
+struct Remote {
+ conn: Connection,
+ session: String,
+}
+
+/// The RemoteDesktop session, opened on the first paste and kept.
+static REMOTE: Mutex<Option<Remote>> = Mutex::new(None);
+/// Refused once: not asked again until Coucou restarts.
+static REFUSED: Mutex<bool> = Mutex::new(false);
+
+fn token_file() -> std::path::PathBuf {
+ super::store::Store::folder().join("remote-desktop-token")
+}
+
+fn variant<T: RefArg + 'static>(v: T) -> Variant<Box<dyn RefArg>> {
+ Variant(Box::new(v))
+}
+
+/// Calls `method` with a request token and waits for its Response (the user
+/// may have a dialog to answer). The results, or why there are none.
+fn request(conn: &Connection, method: &str, args: impl FnOnce(&str) -> dbus::Message, tokens: &mut u32) -> Result<PropMap, String> {
+ *tokens += 1;
+ let token = format!("coucoupaste{}_{}", std::process::id(), tokens);
+ let sender = conn.unique_name().to_string();
+ let path = portal::request_path(&sender, &token);
+ let answer: Arc<Mutex<Option<Result<PropMap, u32>>>> = Arc::new(Mutex::new(None));
+ let slot = answer.clone();
+ let rule = MatchRule::new_signal("org.freedesktop.portal.Request", "Response").with_path(Path::from(path.clone()));
+ let id = conn
+ .add_match(rule, move |(): (), _, msg| {
+ *slot.lock().unwrap() = Some(portal::parse_response(msg));
+ false
+ })
+ .map_err(|e| e.to_string())?;
+ let msg = args(&token);
+ conn.send_with_reply_and_block(msg, Duration::from_secs(10)).map_err(|e| format!("{method}: {e}"))?;
+ let until = Instant::now() + Duration::from_secs(120);
+ while answer.lock().unwrap().is_none() && Instant::now() < until {
+ conn.process(Duration::from_millis(200)).map_err(|e| e.to_string())?;
+ }
+ let _ = conn.remove_match(id);
+ let result = answer.lock().unwrap().take();
+ match result {
+ Some(Ok(results)) => Ok(results),
+ Some(Err(code)) => Err(format!("{method} refused ({code})")),
+ None => Err(format!("{method}: no answer")),
+ }
+}
+
+fn open() -> Result<Remote, String> {
+ let conn = Connection::new_session().map_err(|e| e.to_string())?;
+ let mut tokens = 0;
+ let call = |member: &str| dbus::Message::new_method_call(DEST, OBJECT, IFACE, member).map_err(|e| e.to_string());
+
+ let session_token = format!("coucoupastes{}", std::process::id());
+ let create = call("CreateSession")?;
+ let results = request(&conn, "CreateSession", |token| {
+ let mut options: PropMap = HashMap::new();
+ options.insert("handle_token".into(), variant(token.to_string()));
+ options.insert("session_handle_token".into(), variant(session_token.clone()));
+ create.append1(options)
+ }, &mut tokens)?;
+ let session = portal::session_from_results(&results).ok_or("no session")?;
+
+ let restore = std::fs::read_to_string(token_file()).ok().filter(|t| !t.trim().is_empty());
+ let select = call("SelectDevices")?;
+ request(&conn, "SelectDevices", |token| {
+ let mut options: PropMap = HashMap::new();
+ options.insert("handle_token".into(), variant(token.to_string()));
+ options.insert("types".into(), variant(1u32)); // keyboard
+ options.insert("persist_mode".into(), variant(2u32)); // until revoked
+ if let Some(restore) = &restore {
+ options.insert("restore_token".into(), variant(restore.trim().to_string()));
+ }
+ select.append2(Path::from(session.clone()), options)
+ }, &mut tokens)?;
+
+ let start = call("Start")?;
+ let results = request(&conn, "Start", |token| {
+ let mut options: PropMap = HashMap::new();
+ options.insert("handle_token".into(), variant(token.to_string()));
+ start.append3(Path::from(session.clone()), "", options)
+ }, &mut tokens)?;
+ if let Some(token) = results.get("restore_token").and_then(|v| v.0.as_str()) {
+ let _ = crate::platform::ensure_private_dir(&super::store::Store::folder());
+ let _ = std::fs::write(token_file(), token);
+ }
+ Ok(Remote { conn, session })
+}
+
+fn key(remote: &Remote, keysym: i32, pressed: bool) -> Result<(), String> {
+ let msg = dbus::Message::new_method_call(DEST, OBJECT, IFACE, "NotifyKeyboardKeysym")
+ .map_err(|e| e.to_string())?
+ .append3(Path::from(remote.session.clone()), PropMap::new(), keysym)
+ .append1(pressed as u32);
+ remote.conn.send_with_reply_and_block(msg, Duration::from_secs(5)).map_err(|e| e.to_string())?;
+ Ok(())
+}
+
+/// Ctrl+V (Ctrl+Shift+V) through the RemoteDesktop portal.
+pub fn send_paste(shift: bool) -> Result<(), String> {
+ if *REFUSED.lock().unwrap() {
+ return Err("remote desktop refused".into());
+ }
+ let mut slot = REMOTE.lock().unwrap();
+ if slot.is_none() {
+ match open() {
+ Ok(r) => *slot = Some(r),
+ Err(err) => {
+ *REFUSED.lock().unwrap() = true;
+ return Err(err);
+ }
+ }
+ }
+ let remote = slot.as_ref().unwrap();
+ let mut seq = vec![(XK_CONTROL_L, true)];
+ if shift {
+ seq.push((XK_SHIFT_L, true));
+ }
+ seq.extend([(XK_V, true), (XK_V, false)]);
+ if shift {
+ seq.push((XK_SHIFT_L, false));
+ }
+ seq.push((XK_CONTROL_L, false));
+ for (sym, down) in seq {
+ if let Err(err) = key(remote, sym, down) {
+ // The session ended (the user revoked it): open a new one next time.
+ *slot = None;
+ return Err(err);
+ }
+ }
+ Ok(())
+}
diff --git a/windows/src-tauri/src/lib.rs b/windows/src-tauri/src/lib.rs
index df1b2dc2..b4ea3cb8 100644
--- a/windows/src-tauri/src/lib.rs
+++ b/windows/src-tauri/src/lib.rs
@@ -34,6 +34,7 @@ mod sounds;
mod speech;
mod spotify;
mod tray;
+mod voice;
#[cfg(windows)]
mod webview_drop;
@@ -539,6 +540,9 @@ fn secret_set(app: AppHandle, key: String, value: String) -> Result<(), String>
}
let before = (key == "github-token").then(|| secrets::get(&key));
secrets::set(&key, &value)?;
+ if key == claude::KEY {
+ voice::sync(&app);
+ }
if let Some(before) = before {
if secrets::get(&key) != before {
integrations::github_token_changed(&app);
@@ -551,6 +555,9 @@ fn secret_set(app: AppHandle, key: String, value: String) -> Result<(), String>
fn secret_clear(app: AppHandle, key: String) -> Result<(), String> {
let had = key == "github-token" && secrets::present(&key);
secrets::clear(&key)?;
+ if key == claude::KEY {
+ voice::sync(&app);
+ }
if had {
integrations::github_token_changed(&app);
}
@@ -853,6 +860,8 @@ pub fn run() {
notes::notes_retry,
notes::notes_markdown,
notes::notes_export,
+ voice::voice_done,
+ voice::voice_say_native,
])
.setup(move |app| {
let handle = app.handle().clone();
diff --git a/windows/src-tauri/src/portal.rs b/windows/src-tauri/src/portal.rs
index cc8aa9e5..9854e46c 100644
--- a/windows/src-tauri/src/portal.rs
+++ b/windows/src-tauri/src/portal.rs
@@ -236,6 +236,8 @@ pub enum Event {
Unavailable(String),
/// A shortcut was pressed.
Activated(String),
+ /// A shortcut was released (Flow dictation's hold-to-talk key).
+ Deactivated(String),
}
enum Command {
@@ -519,11 +521,11 @@ impl Portal {
};
match (&*iface, &*member) {
(REQUEST, "Response") => self.response(&path, msg, on_event),
- (IFACE, "Activated") => {
+ (IFACE, member @ ("Activated" | "Deactivated")) => {
let Phase::Bound { session, .. } = &self.phase else { return };
if let Ok((from, id)) = msg.read2::<Path, String>() {
if &*from == session.as_str() {
- on_event(Event::Activated(id));
+ on_event(if member == "Activated" { Event::Activated(id) } else { Event::Deactivated(id) });
}
}
}
diff --git a/windows/src-tauri/src/secrets.rs b/windows/src-tauri/src/secrets.rs
index e81ef527..6f065a45 100644
--- a/windows/src-tauri/src/secrets.rs
+++ b/windows/src-tauri/src/secrets.rs
@@ -20,6 +20,8 @@ pub const KNOWN_KEYS: &[&str] = &[
"resend-api-key",
"notion-api-key",
"calcom-api-key",
+ // « OK Coucou »'s answers in an ElevenLabs voice.
+ "elevenlabs-api-key",
];
fn entry(key: &str) -> Option<Entry> {
diff --git a/windows/src-tauri/src/speech/mod.rs b/windows/src-tauri/src/speech/mod.rs
index 950e2171..b7c269a9 100644
--- a/windows/src-tauri/src/speech/mod.rs
+++ b/windows/src-tauri/src/speech/mod.rs
@@ -165,6 +165,7 @@ pub fn is_ready() -> bool {
fn emit_status(app: &AppHandle) {
let _ = app.emit("speech-models", status());
+ crate::voice::sync(app);
}
/// Downloads what is missing, with the progress in `speech-models` events.
diff --git a/windows/src-tauri/src/voice/brain.rs b/windows/src-tauri/src/voice/brain.rs
new file mode 100644
index 00000000..71e70296
--- /dev/null
+++ b/windows/src-tauri/src/voice/brain.rs
@@ -0,0 +1,228 @@
+// « OK Coucou »'s brain: Claude with the user's key and Coucou's tools —
+// pills, music, the user's own data, opening an app, preparing an email (the
+// mail app opens with it filled in; the user sends it) and Anthropic's web
+// search. The conversation lives in memory only and is forgotten 90 s after
+// the last turn. Port of ClaudeVoiceBrain.swift.
+
+use std::time::{Duration, Instant};
+
+use serde_json::{json, Value};
+use tauri::{AppHandle, Manager};
+
+const FAST_MODEL: &str = "claude-haiku-4-5";
+const FORGET_AFTER: Duration = Duration::from_secs(90);
+
+pub struct Brain {
+ messages: Vec<Value>,
+ last: Instant,
+}
+
+impl Default for Brain {
+ fn default() -> Self {
+ Self { messages: Vec::new(), last: Instant::now() }
+ }
+}
+
+pub struct Reply {
+ pub text: String,
+ pub acted: bool,
+}
+
+fn tools() -> Vec<Value> {
+ vec![
+ json!({
+ "name": "pills",
+ "description": "Add, remove, set as main or list the pills shown in the island at the top of the screen. At most 4 pills besides the main one: remove one first if needed.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "action": { "type": "string", "enum": ["add", "remove", "set_main", "list"] },
+ "names": { "type": "array", "items": { "type": "string" }, "description": "Pill names as listed (e.g. Stripe, GitHub, Cursor). Empty for list." }
+ },
+ "required": ["action"]
+ }
+ }),
+ json!({
+ "name": "music",
+ "description": "Control Spotify: play or pause, next, previous, set the volume, or search a song, an artist or a playlist in Spotify.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "action": { "type": "string", "enum": ["play_pause", "next", "previous", "set_volume", "search"] },
+ "query": { "type": "string", "description": "What to search, for search." },
+ "volume": { "type": "integer", "description": "0-100 for set_volume." }
+ },
+ "required": ["action"]
+ }
+ }),
+ json!({
+ "name": "service_info",
+ "description": "Read the user's real data that Coucou already has, as JSON: the active pills, the agent sessions (Claude Code, Codex…) and what they are doing, Claude or Codex plan usage, Spotify now playing, and the services' latest data (Stripe, GitHub, Vercel, Resend, n8n, Notion, Cal.com).",
+ "input_schema": { "type": "object", "properties": {} }
+ }),
+ json!({
+ "name": "open_app",
+ "description": "Open an app on this computer by name (Firefox, VS Code, Notion, Spotify…).",
+ "input_schema": { "type": "object", "properties": { "name": { "type": "string" } }, "required": ["name"] }
+ }),
+ json!({
+ "name": "prepare_email",
+ "description": "Open the user's mail app with an email filled in, for them to check and send (it is never sent automatically). The recipient must be an email address.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "recipient": { "type": "string", "description": "An email address." },
+ "subject": { "type": "string" },
+ "body": { "type": "string", "description": "The full email text, written for the user." }
+ },
+ "required": ["recipient", "subject", "body"]
+ }
+ }),
+ json!({ "type": "web_search_20250305", "name": "web_search", "max_uses": 3 }),
+ ]
+}
+
+fn system_prompt(pills: &str) -> String {
+ let t = crate::platform::local_time();
+ let os = if cfg!(windows) { "Windows" } else { "Linux" };
+ format!(
+ "You are Coucou, a voice assistant living at the top of the screen of the user's {os} computer, a bit like Jarvis: \
+quick, warm, a little playful. It is {:04}-{:02}-{:02} {:02}:{:02}.\n\
+Everything you write is read aloud: answer in the language the user spoke, in one or two short sentences, \
+no markdown, no lists, no emojis, no links.\n\
+Act with the tools instead of describing: pills, music, service_info for the user's own data, open_app, \
+prepare_email, and web_search for anything current (weather, news, scores, prices, facts you are not sure of). \
+Never invent the user's data.\n\
+Email: gather the recipient's address, the subject and what to say (write the text yourself from what the user \
+wants to say, in the language they used), asking for what is missing one short question at a time, then call \
+prepare_email. The email is never sent by you: the user checks it in their mail app and sends it.\n\
+When you need an answer from the user, end with one short question. Otherwise never end with a question.\n\
+Pills: {pills}",
+ t.year, t.month, t.day, t.hour, t.minute
+ )
+}
+
+impl Brain {
+ pub fn respond(&mut self, app: &AppHandle, transcript: &str) -> Option<Reply> {
+ if self.last.elapsed() > FORGET_AFTER {
+ self.messages.clear();
+ }
+ self.last = Instant::now();
+ self.messages.push(json!({ "role": "user", "content": transcript }));
+ self.trim();
+ let pills = super::ask_island(app, "pills", json!({ "action": "list" })).unwrap_or_default();
+ let system = system_prompt(&pills);
+ let tools = tools();
+ let chat_model = app
+ .try_state::<crate::Shared>()
+ .map(|s| s.settings.lock().unwrap().model.clone())
+ .unwrap_or_default();
+ let mut acted = false;
+ let mut spoken = String::new();
+ for _ in 0..5 {
+ let turn = tauri::async_runtime::block_on(async {
+ match crate::claude::voice_turn(FAST_MODEL, &system, &self.messages, &tools).await {
+ Ok(v) => Ok(v),
+ Err(err) if !chat_model.is_empty() => {
+ crate::log::line(format!("voice: {err}; the chat's model instead"));
+ crate::claude::voice_turn(&chat_model, &system, &self.messages, &tools).await
+ }
+ Err(err) => Err(err),
+ }
+ });
+ let turn = match turn {
+ Ok(t) => t,
+ Err(err) => {
+ crate::log::line(format!("voice: Claude turn failed: {err}"));
+ if self.messages.last().is_some_and(|m| m["content"].is_string()) {
+ self.messages.pop();
+ }
+ return acted.then(|| Reply { text: String::new(), acted });
+ }
+ };
+ let content = turn["content"].as_array().cloned().unwrap_or_default();
+ self.messages.push(json!({ "role": "assistant", "content": content.clone() }));
+ let text: String = content.iter().filter(|b| b["type"] == "text").filter_map(|b| b["text"].as_str()).collect::<Vec<_>>().join(" ");
+ if !text.trim().is_empty() {
+ spoken = text;
+ }
+ let stop = turn["stop_reason"].as_str().unwrap_or("end_turn");
+ if stop == "pause_turn" {
+ continue;
+ }
+ let uses: Vec<&Value> = content.iter().filter(|b| b["type"] == "tool_use").collect();
+ if stop != "tool_use" || uses.is_empty() {
+ break;
+ }
+ let mut results = Vec::new();
+ for u in uses {
+ let (result, did) = super::run_tool(app, u["name"].as_str().unwrap_or(""), &u["input"]);
+ acted |= did;
+ results.push(json!({ "type": "tool_result", "tool_use_id": u["id"], "content": result }));
+ }
+ self.messages.push(json!({ "role": "user", "content": results }));
+ spoken.clear();
+ }
+ self.repair();
+ Some(Reply { text: spoken_text(&spoken), acted })
+ }
+
+ /// Keeps the last exchanges, never cutting between a tool call and its result.
+ fn trim(&mut self) {
+ if self.messages.len() <= 24 {
+ return;
+ }
+ let mut cut = self.messages.len() - 24;
+ while cut < self.messages.len() && !(self.messages[cut]["role"] == "user" && self.messages[cut]["content"].is_string()) {
+ cut += 1;
+ }
+ self.messages.drain(..cut.min(self.messages.len() - 1));
+ }
+
+ /// The API refuses a history ending on a tool call with no result after it.
+ fn repair(&mut self) {
+ let Some(last) = self.messages.last() else { return };
+ if last["role"] != "assistant" {
+ return;
+ }
+ let calls = last["content"].as_array().is_some_and(|c| c.iter().any(|b| b["type"] == "tool_use"));
+ if calls {
+ self.messages.pop();
+ }
+ }
+}
+
+/// What Claude wrote, ready to be said: citations and markdown left out.
+pub fn spoken_text(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ let mut skip = 0;
+ for c in text.chars() {
+ match c {
+ '[' => skip += 1,
+ ']' if skip > 0 => skip -= 1,
+ '*' | '#' | '`' | '_' if skip == 0 => {}
+ _ if skip == 0 => out.push(c),
+ _ => {}
+ }
+ }
+ out.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn markdown_and_citations_are_not_read_aloud() {
+ assert_eq!(spoken_text("**Done** — Stripe [1] is in the island."), "Done — Stripe is in the island.");
+ assert_eq!(spoken_text(" Il fait 20 °C à Bordeaux. "), "Il fait 20 °C à Bordeaux.");
+ }
+
+ #[test]
+ fn every_tool_has_a_schema() {
+ for tool in tools() {
+ assert!(tool["name"].is_string());
+ assert!(tool.get("input_schema").is_some() || tool["type"] == "web_search_20250305");
+ }
+ }
+}
diff --git a/windows/src-tauri/src/voice/mod.rs b/windows/src-tauri/src/voice/mod.rs
new file mode 100644
index 00000000..c5adef88
--- /dev/null
+++ b/windows/src-tauri/src/voice/mod.rs
@@ -0,0 +1,566 @@
+// « OK Coucou » on Windows and Linux: say it, then ask anything. Port of the
+// Mac's Voice/ (VoiceEngine.swift and its friends).
+//
+// While it is on, the microphone is open and a cheap energy detector cuts what
+// you say into stretches; only a stretch of speech goes to Parakeet, on this
+// computer, and only a stretch that starts with « OK Coucou » goes further:
+// what you said after it goes to Claude with your own key (brain.rs), never
+// the audio. Mochi shows it all in the pill at the bottom of the screen —
+// listening, thinking, then the answer, said out loud by the system's voice
+// (the pill's speechSynthesis) or ElevenLabs with your key.
+//
+// It stops listening while you dictate, while a call is recorded and while it
+// speaks itself.
+
+pub mod brain;
+pub mod wake;
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::mpsc;
+use std::sync::{Arc, Mutex, OnceLock};
+use std::time::{Duration, Instant};
+
+use serde::Serialize;
+use serde_json::{json, Value};
+use tauri::{AppHandle, Emitter};
+
+use crate::flow::pill;
+use crate::speech::{self, audio};
+
+/// What the pill shows of a conversation.
+#[derive(Debug, Clone, Serialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct VoiceView {
+ /// "listening", "thinking", "speaking".
+ pub phase: String,
+ /// What you said (after « OK Coucou »).
+ pub heard: String,
+ /// Coucou's answer.
+ pub answer: String,
+}
+
+struct Running {
+ stop: Arc<AtomicBool>,
+}
+
+static RUNNING: Mutex<Option<Running>> = Mutex::new(None);
+
+/// Rust waits on these for the island's answers and the end of speech.
+fn pending() -> &'static Mutex<HashMap<u64, mpsc::Sender<String>>> {
+ static P: OnceLock<Mutex<HashMap<u64, mpsc::Sender<String>>>> = OnceLock::new();
+ P.get_or_init(|| Mutex::new(HashMap::new()))
+}
+static NEXT_ID: AtomicU64 = AtomicU64::new(1);
+
+/// Whether « OK Coucou » can run: the voice model and a Claude key.
+pub fn requirements_met() -> bool {
+ speech::is_ready() && crate::secrets::present(crate::claude::KEY)
+}
+
+/// Starts or stops listening to follow the preference.
+pub fn sync(app: &AppHandle) {
+ let wanted = crate::flow::get(app).is_some_and(|f| f.store.lock().unwrap().prefs.voice);
+ let mut running = RUNNING.lock().unwrap();
+ match (wanted && requirements_met(), running.is_some()) {
+ (true, false) => {
+ let stop = Arc::new(AtomicBool::new(false));
+ let app2 = app.clone();
+ let stop2 = stop.clone();
+ std::thread::Builder::new().name("coucou-voice".into()).spawn(move || run(app2, stop2)).ok();
+ *running = Some(Running { stop });
+ speech::preload();
+ crate::log::line("voice: « OK Coucou » listening");
+ }
+ (false, true) => {
+ if let Some(r) = running.take() {
+ r.stop.store(true, Ordering::Relaxed);
+ }
+ let dictation = crate::flow::get(app).is_some_and(|f| f.store.lock().unwrap().prefs.enabled);
+ if !dictation && !crate::notes::is_recording(app) {
+ speech::unload();
+ }
+ crate::log::line("voice: « OK Coucou » off");
+ }
+ _ => {}
+ }
+}
+
+/// « OK Coucou » is listening (the voice model stays loaded).
+pub fn is_listening() -> bool {
+ RUNNING.lock().unwrap().is_some()
+}
+
+/// Starts again on another microphone.
+pub fn restart(app: &AppHandle) {
+ if let Some(r) = RUNNING.lock().unwrap().take() {
+ r.stop.store(true, Ordering::Relaxed);
+ }
+ sync(app);
+}
+
+fn show(app: &AppHandle, view: Option<VoiceView>) {
+ pill::update(app, |s| s.voice = view);
+}
+
+enum Mode {
+ /// Waiting for « OK Coucou ».
+ Wake,
+ /// Heard « OK Coucou » alone: what comes next is the question.
+ Command { text: String, until: Instant },
+ /// Coucou asked something back: an answer comes without « OK Coucou ».
+ FollowUp { until: Instant },
+}
+
+fn run(app: AppHandle, stop: Arc<AtomicBool>) {
+ let mic = crate::flow::get(&app).map(|f| f.store.lock().unwrap().prefs.mic.clone()).unwrap_or_default();
+ let (tx, rx) = mpsc::channel::<Vec<f32>>();
+ let capture = audio::Capture::start(audio::Source::Mic(if mic.is_empty() { None } else { Some(mic) }), move |s| {
+ let _ = tx.send(s.to_vec());
+ });
+ let mut capture = match capture {
+ Ok(c) => c,
+ Err(err) => {
+ crate::log::line(format!("voice: microphone: {err}"));
+ *RUNNING.lock().unwrap() = None;
+ return;
+ }
+ };
+ let mut vad = wake::Vad::default();
+ let mut mode = Mode::Wake;
+ let mut brain = brain::Brain::default();
+ let mut hide_at: Option<Instant> = None;
+ while !stop.load(Ordering::Relaxed) {
+ let chunk = match rx.recv_timeout(Duration::from_millis(300)) {
+ Ok(c) => Some(c),
+ Err(mpsc::RecvTimeoutError::Timeout) => None,
+ Err(mpsc::RecvTimeoutError::Disconnected) => break,
+ };
+ let now = Instant::now();
+ // The pill's last answer goes away after a while.
+ if hide_at.is_some_and(|t| now > t) {
+ hide_at = None;
+ show(&app, None);
+ }
+ match &mode {
+ Mode::Command { until, .. } | Mode::FollowUp { until } if now > *until => {
+ mode = Mode::Wake;
+ vad.hangover = 25;
+ show(&app, None);
+ }
+ _ => {}
+ }
+ // Dictation and calls have the microphone: Coucou doesn't listen meanwhile.
+ if pill::dictating(&app) || crate::notes::is_recording(&app) {
+ vad.reset();
+ continue;
+ }
+ let Some(chunk) = chunk else { continue };
+ for segment in vad.feed(&chunk) {
+ if segment.len() < speech::SAMPLE_RATE as usize * 35 / 100 {
+ continue;
+ }
+ let Some(heard) = speech::transcribe(&segment).map(|t| t.text.trim().to_string()) else { continue };
+ if heard.is_empty() {
+ continue;
+ }
+ let question = match &mut mode {
+ Mode::Wake => match wake::after_wake(&heard) {
+ Some(rest) => {
+ crate::log::line("voice: « OK Coucou » heard");
+ hide_at = None;
+ if crate::flow::text::word_count(&rest) >= 2 {
+ Some(rest)
+ } else {
+ show(&app, Some(VoiceView { phase: "listening".into(), heard: rest.clone(), answer: String::new() }));
+ mode = Mode::Command { text: rest, until: now + Duration::from_secs(7) };
+ // A question may come with pauses in it.
+ vad.hangover = 40;
+ None
+ }
+ }
+ None => None,
+ },
+ Mode::Command { text, .. } => {
+ let full = format!("{text} {heard}").trim().to_string();
+ Some(full)
+ }
+ Mode::FollowUp { .. } => Some(heard),
+ };
+ let Some(question) = question else { continue };
+ mode = Mode::Wake;
+ vad.hangover = 25;
+ show(&app, Some(VoiceView { phase: "thinking".into(), heard: question.clone(), answer: String::new() }));
+ let reply = brain.respond(&app, &question);
+ let answer = reply.as_ref().map(|r| r.text.clone()).unwrap_or_default();
+ if answer.is_empty() {
+ let done = reply.is_some_and(|r| r.acted);
+ show(&app, Some(VoiceView {
+ phase: "speaking".into(),
+ heard: question,
+ answer: if done { "✓".into() } else { crate::i18n::t("Coucou couldn't answer: check your Claude key in Notch → Chat.") },
+ }));
+ hide_at = Some(Instant::now() + Duration::from_secs(5));
+ continue;
+ }
+ show(&app, Some(VoiceView { phase: "speaking".into(), heard: question.clone(), answer: answer.clone() }));
+ speak(&app, &answer);
+ vad.reset();
+ // Drop what the microphone heard of Coucou's own voice.
+ while rx.try_recv().is_ok() {}
+ if answer.trim_end().ends_with('?') {
+ mode = Mode::FollowUp { until: Instant::now() + Duration::from_secs(8) };
+ vad.hangover = 40;
+ show(&app, Some(VoiceView { phase: "listening".into(), heard: String::new(), answer }));
+ } else {
+ hide_at = Some(Instant::now() + Duration::from_secs(6));
+ }
+ }
+ }
+ capture.stop();
+ show(&app, None);
+}
+
+// ── Speaking ──────────────────────────────────────────────────────────────────
+
+#[derive(Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+struct Say {
+ id: u64,
+ text: String,
+ lang: Option<&'static str>,
+ /// ElevenLabs' MP3, base64.
+ audio: Option<String>,
+ gender: String,
+}
+
+/// Says `text` and returns once it is said (or after a sensible time).
+fn speak(app: &AppHandle, text: &str) {
+ let (output, gender) = crate::flow::get(app)
+ .map(|f| {
+ let p = &f.store.lock().unwrap().prefs;
+ (p.voice_output.clone(), p.voice_gender.clone())
+ })
+ .unwrap_or_else(|| ("system".into(), "female".into()));
+ if output == "off" {
+ std::thread::sleep(Duration::from_millis(1500));
+ return;
+ }
+ let audio = if output == "elevenlabs" { elevenlabs(text, &gender).map(|mp3| base64(&mp3)) } else { None };
+ let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
+ let (tx, rx) = mpsc::channel();
+ pending().lock().unwrap().insert(id, tx);
+ let _ = app.emit_to(pill::LABEL, "voice-say", Say { id, text: text.into(), lang: wake::speech_lang(text), audio, gender });
+ let words = crate::flow::text::word_count(text) as u64;
+ let _ = rx.recv_timeout(Duration::from_millis(4000 + words * 600));
+ pending().lock().unwrap().remove(&id);
+}
+
+/// The pill says it has finished speaking (`id`), or the island answers a question.
+#[tauri::command]
+pub fn voice_done(id: u64, text: Option<String>) {
+ if let Some(tx) = pending().lock().unwrap().remove(&id) {
+ let _ = tx.send(text.unwrap_or_default());
+ }
+}
+
+/// Where the webview has no voice (WebKitGTK often): speech-dispatcher or
+/// espeak-ng, when installed.
+#[tauri::command]
+pub async fn voice_say_native(text: String, lang: Option<String>) -> bool {
+ tauri::async_runtime::spawn_blocking(move || {
+ #[cfg(target_os = "linux")]
+ {
+ let lang = lang.unwrap_or_default();
+ let short = lang.split('-').next().unwrap_or("").to_string();
+ if let Some(spd) = crate::platform::find_on_path("spd-say") {
+ let mut cmd = std::process::Command::new(spd);
+ cmd.arg("-w");
+ if !short.is_empty() {
+ cmd.args(["-l", &short]);
+ }
+ return cmd.arg(&text).status().is_ok_and(|s| s.success());
+ }
+ if let Some(espeak) = crate::platform::find_on_path("espeak-ng") {
+ let mut cmd = std::process::Command::new(espeak);
+ if !short.is_empty() {
+ cmd.args(["-v", &short]);
+ }
+ return cmd.arg(&text).status().is_ok_and(|s| s.success());
+ }
+ false
+ }
+ #[cfg(windows)]
+ {
+ let _ = (text, lang);
+ false
+ }
+ })
+ .await
+ .unwrap_or(false)
+}
+
+/// ElevenLabs' MP3 for `text`, with the user's key; None and the system voice speaks.
+fn elevenlabs(text: &str, gender: &str) -> Option<Vec<u8>> {
+ let key = crate::secrets::get("elevenlabs-api-key")?;
+ tauri::async_runtime::block_on(async {
+ let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().ok()?;
+ let voices: Value = client.get("https://api.elevenlabs.io/v1/voices").header("xi-api-key", &key).send().await.ok()?.json().await.ok()?;
+ let voice = pick_voice(voices["voices"].as_array()?, gender)?;
+ let response = client
+ .post(format!("https://api.elevenlabs.io/v1/text-to-speech/{voice}?output_format=mp3_44100_128"))
+ .header("xi-api-key", &key)
+ .header("accept", "audio/mpeg")
+ .json(&json!({ "text": text, "model_id": "eleven_flash_v2_5" }))
+ .send()
+ .await
+ .ok()?;
+ if !response.status().is_success() {
+ crate::log::line(format!("voice: ElevenLabs {}", response.status()));
+ return None;
+ }
+ Some(response.bytes().await.ok()?.to_vec())
+ })
+}
+
+/// First English voice of the gender (a young one first for a man); else any
+/// voice of the gender; else the first one. Same choice as the Mac.
+pub fn pick_voice(voices: &[Value], gender: &str) -> Option<String> {
+ let label = |v: &Value, k: &str| v["labels"][k].as_str().unwrap_or("").to_lowercase();
+ let english = |v: &Value| {
+ let lang = label(v, "language");
+ if !lang.is_empty() {
+ return lang.starts_with("en");
+ }
+ let accent = label(v, "accent");
+ accent.is_empty() || ["american", "british", "australian", "irish", "english", "us", "uk"].iter().any(|a| accent.contains(a))
+ };
+ let of_gender: Vec<&Value> = voices.iter().filter(|v| label(v, "gender") == gender).collect();
+ let english_ones: Vec<&&Value> = of_gender.iter().filter(|v| english(v)).collect();
+ let young = if gender == "male" { english_ones.iter().find(|v| label(v, "age").contains("young")).copied() } else { None };
+ let v = young.or_else(|| english_ones.first().copied()).copied().or_else(|| of_gender.first().copied()).or_else(|| voices.first())?;
+ v["voice_id"].as_str().map(String::from)
+}
+
+fn base64(bytes: &[u8]) -> String {
+ const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
+ for chunk in bytes.chunks(3) {
+ let b = [chunk[0], *chunk.get(1).unwrap_or(&0), *chunk.get(2).unwrap_or(&0)];
+ let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32;
+ out.push(T[(n >> 18) as usize & 63] as char);
+ out.push(T[(n >> 12) as usize & 63] as char);
+ out.push(if chunk.len() > 1 { T[(n >> 6) as usize & 63] as char } else { '=' });
+ out.push(if chunk.len() > 2 { T[n as usize & 63] as char } else { '=' });
+ }
+ out
+}
+
+// ── The tools ─────────────────────────────────────────────────────────────────
+
+/// Asks the island (it knows the pills and the services' data) and waits for its answer.
+pub fn ask_island(app: &AppHandle, kind: &str, input: Value) -> Option<String> {
+ let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
+ let (tx, rx) = mpsc::channel();
+ pending().lock().unwrap().insert(id, tx);
+ let _ = app.emit_to(crate::island::WINDOW_LABEL, "voice-ask", json!({ "id": id, "kind": kind, "input": input }));
+ let answer = rx.recv_timeout(Duration::from_secs(5)).ok();
+ pending().lock().unwrap().remove(&id);
+ answer
+}
+
+/// Runs one of Claude's tools: what goes back to Claude, and whether something changed.
+pub fn run_tool(app: &AppHandle, name: &str, input: &Value) -> (String, bool) {
+ match name {
+ "pills" => {
+ let answer = ask_island(app, "pills", input.clone()).unwrap_or_else(|| "the island did not answer".into());
+ let acted = input["action"].as_str() != Some("list") && !answer.contains("unknown");
+ (answer, acted)
+ }
+ "service_info" => (ask_island(app, "info", json!({})).unwrap_or_else(|| "no data".into()), false),
+ "music" => music(app, input),
+ "open_app" => open_app(input["name"].as_str().unwrap_or("")),
+ "prepare_email" => {
+ let to = input["recipient"].as_str().unwrap_or("").trim().to_string();
+ if !to.contains('@') {
+ return ("No email address: ask the user to spell it.".into(), false);
+ }
+ let enc = |s: &str| -> String {
+ s.bytes()
+ .map(|b| match b {
+ b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
+ _ => format!("%{b:02X}"),
+ })
+ .collect()
+ };
+ let url = format!(
+ "mailto:{}?subject={}&body={}",
+ enc(&to).replace("%40", "@"),
+ enc(input["subject"].as_str().unwrap_or("")),
+ enc(input["body"].as_str().unwrap_or(""))
+ );
+ crate::platform::open_url(&url);
+ (format!("The mail app opened with the email to {to}. Nothing is sent until the user sends it."), true)
+ }
+ _ => (format!("unknown tool {name}"), false),
+ }
+}
+
+fn music(app: &AppHandle, input: &Value) -> (String, bool) {
+ let action = input["action"].as_str().unwrap_or("");
+ let (control, value) = match action {
+ "play_pause" => ("playPause", None),
+ "next" => ("next", None),
+ "previous" => ("previous", None),
+ "set_volume" => ("volume", Some(input["volume"].as_f64().unwrap_or(50.0).clamp(0.0, 100.0))),
+ "search" => {
+ let q = input["query"].as_str().unwrap_or("").trim();
+ if q.is_empty() {
+ return ("missing query".into(), false);
+ }
+ let encoded: String = q.chars().map(|c| if c.is_alphanumeric() { c.to_string() } else { "%20".into() }).collect();
+ crate::platform::open_url(&format!("spotify:search:{encoded}"));
+ return (format!("Spotify opened on a search for {q}: the user picks what to play."), true);
+ }
+ _ => return ("unknown action".into(), false),
+ };
+ let app2 = app.clone();
+ let done = tauri::async_runtime::block_on(crate::spotify::spotify_control(app2, control.into(), value));
+ if done {
+ ("done".into(), true)
+ } else {
+ ("Spotify is not running or not playing: tell the user to open it.".into(), false)
+ }
+}
+
+/// Opens an app whose name matches: a Start menu shortcut on Windows, a
+/// .desktop entry on Linux. Nothing else is ever run.
+fn open_app(name: &str) -> (String, bool) {
+ let wanted = wake::normalize(name);
+ if wanted.is_empty() {
+ return ("no name".into(), false);
+ }
+ match find_app(&wanted) {
+ Some((label, launch)) => {
+ launch();
+ (format!("{label} opened"), true)
+ }
+ None => (format!("No app named {name} on this computer."), false),
+ }
+}
+
+#[cfg(windows)]
+fn find_app(wanted: &str) -> Option<(String, Box<dyn FnOnce()>)> {
+ let mut roots = Vec::new();
+ if let Some(p) = std::env::var_os("APPDATA") {
+ roots.push(std::path::PathBuf::from(p).join("Microsoft\\Windows\\Start Menu\\Programs"));
+ }
+ if let Some(p) = std::env::var_os("ProgramData") {
+ roots.push(std::path::PathBuf::from(p).join("Microsoft\\Windows\\Start Menu\\Programs"));
+ }
+ let mut best: Option<(usize, std::path::PathBuf)> = None;
+ let mut stack = roots;
+ while let Some(dir) = stack.pop() {
+ let Ok(entries) = std::fs::read_dir(&dir) else { continue };
+ for e in entries.flatten() {
+ let path = e.path();
+ if path.is_dir() {
+ stack.push(path);
+ } else if path.extension().is_some_and(|x| x.eq_ignore_ascii_case("lnk")) {
+ let stem = wake::normalize(&path.file_stem().unwrap_or_default().to_string_lossy());
+ let score = if stem == wanted { 0 } else if stem.starts_with(wanted) { 1 } else if stem.contains(wanted) { 2 } else { continue };
+ if best.as_ref().is_none_or(|(s, p)| score < *s || (score == *s && path.as_os_str().len() < p.as_os_str().len())) {
+ best = Some((score, path));
+ }
+ }
+ }
+ }
+ let (_, path) = best?;
+ let label = path.file_stem()?.to_string_lossy().to_string();
+ Some((label, Box::new(move || crate::platform::reveal_folder(&path.to_string_lossy()))))
+}
+
+#[cfg(target_os = "linux")]
+fn find_app(wanted: &str) -> Option<(String, Box<dyn FnOnce()>)> {
+ let mut dirs = vec![std::path::PathBuf::from("/usr/share/applications"), std::path::PathBuf::from("/var/lib/flatpak/exports/share/applications")];
+ dirs.push(crate::platform::home_dir().join(".local/share/applications"));
+ dirs.push(crate::platform::home_dir().join(".local/share/flatpak/exports/share/applications"));
+ let mut best: Option<(usize, String, String)> = None;
+ for dir in dirs {
+ let Ok(entries) = std::fs::read_dir(&dir) else { continue };
+ for e in entries.flatten() {
+ let path = e.path();
+ if path.extension().is_none_or(|x| x != "desktop") {
+ continue;
+ }
+ let Ok(text) = std::fs::read_to_string(&path) else { continue };
+ if text.lines().any(|l| l.trim() == "NoDisplay=true") {
+ continue;
+ }
+ let Some(app_name) = text.lines().find_map(|l| l.strip_prefix("Name=")).map(str::trim) else { continue };
+ let n = wake::normalize(app_name);
+ let score = if n == wanted { 0 } else if n.starts_with(wanted) { 1 } else if n.contains(wanted) { 2 } else { continue };
+ let id = path.file_stem().unwrap_or_default().to_string_lossy().to_string();
+ if best.as_ref().is_none_or(|(s, _, _)| score < *s) {
+ best = Some((score, app_name.to_string(), id));
+ }
+ }
+ }
+ let (_, label, id) = best?;
+ Some((label, Box::new(move || {
+ let _ = std::process::Command::new("gtk-launch").arg(&id).spawn();
+ })))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn base64_matches_the_standard() {
+ assert_eq!(base64(b"Man"), "TWFu");
+ assert_eq!(base64(b"Ma"), "TWE=");
+ assert_eq!(base64(b"M"), "TQ==");
+ assert_eq!(base64(b""), "");
+ }
+
+ #[test]
+ fn the_voice_is_picked_like_on_the_mac() {
+ let voices: Vec<Value> = serde_json::from_str(r#"[
+ {"voice_id":"a","labels":{"gender":"male","accent":"american","age":"middle aged"}},
+ {"voice_id":"b","labels":{"gender":"male","accent":"british","age":"young"}},
+ {"voice_id":"c","labels":{"gender":"female","accent":"american"}},
+ {"voice_id":"d","labels":{"gender":"female","language":"fr"}}
+ ]"#).unwrap();
+ assert_eq!(pick_voice(&voices, "male").as_deref(), Some("b"));
+ assert_eq!(pick_voice(&voices, "female").as_deref(), Some("c"));
+ assert_eq!(pick_voice(&[], "female"), None);
+ }
+}
+
+/// With the models downloaded: `COUCOU_TEST_WAKE_WAV=… cargo test -- --ignored wake_in`.
+#[cfg(test)]
+mod with_models {
+ #[test]
+ #[ignore]
+ fn hears_the_wake_phrase_in_a_recording() {
+ let path = std::env::var("COUCOU_TEST_WAKE_WAV").expect("COUCOU_TEST_WAKE_WAV");
+ let bytes = std::fs::read(&path).unwrap();
+ let rate = u32::from_le_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]);
+ let pcm: Vec<f32> = bytes[44..].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0).collect();
+ let mut r = crate::speech::audio::Resampler::new(rate);
+ let mut samples = Vec::new();
+ r.process(&pcm, &mut samples);
+ let mut vad = super::wake::Vad::default();
+ let mut heard = Vec::new();
+ for chunk in samples.chunks(1600) {
+ for segment in vad.feed(chunk) {
+ let text = crate::speech::transcribe(&segment).expect("models").text;
+ println!("segment {:.2} s: {text}", segment.len() as f32 / 16_000.0);
+ heard.push(super::wake::after_wake(&text));
+ }
+ }
+ println!("{heard:?}");
+ assert!(heard.iter().any(|h| h.as_deref().is_some_and(|q| !q.is_empty())));
+ }
+}
diff --git a/windows/src-tauri/src/voice/wake.rs b/windows/src-tauri/src/voice/wake.rs
new file mode 100644
index 00000000..6bb168f9
--- /dev/null
+++ b/windows/src-tauri/src/voice/wake.rs
@@ -0,0 +1,229 @@
+// « OK Coucou »: hearing the wake phrase in what Parakeet wrote, and telling
+// speech from silence on the microphone. Nothing here touches the OS.
+
+/// Lower case, accents gone, everything but letters and digits a space.
+pub fn normalize(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ for c in text.chars().flat_map(char::to_lowercase) {
+ let base = match c {
+ 'à' | 'á' | 'â' | 'ä' | 'ã' | 'å' => 'a',
+ 'ç' => 'c',
+ 'è' | 'é' | 'ê' | 'ë' => 'e',
+ 'ì' | 'í' | 'î' | 'ï' => 'i',
+ 'ò' | 'ó' | 'ô' | 'ö' | 'õ' => 'o',
+ 'ù' | 'ú' | 'û' | 'ü' => 'u',
+ 'ÿ' | 'ý' => 'y',
+ 'ñ' => 'n',
+ c if c.is_alphanumeric() => c,
+ _ => ' ',
+ };
+ out.push(base);
+ }
+ out.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+/// How Parakeet may write "OK".
+fn is_ok(word: &str) -> bool {
+ matches!(word, "ok" | "okay" | "oke" | "okey" | "hey")
+}
+
+/// How Parakeet may write "coucou", in one word.
+fn is_coucou(word: &str) -> bool {
+ matches!(
+ word,
+ "coucou" | "coucous" | "cuckoo" | "cucu" | "kuku" | "koukou" | "cookoo" | "coocoo" | "cocou" | "kukuu" | "couscous"
+ )
+}
+
+/// …or in two.
+fn is_coucou_pair(a: &str, b: &str) -> bool {
+ matches!((a, b), ("cou", "cou") | ("coo", "coo") | ("ku", "ku") | ("cu", "cu") | ("koo", "koo") | ("cuck", "oo"))
+}
+
+/// When `text` holds the wake phrase, what was said after it (maybe nothing).
+/// The words keep their case and punctuation, as Parakeet wrote them.
+pub fn after_wake(text: &str) -> Option<String> {
+ let words: Vec<&str> = text.split_whitespace().collect();
+ let norm: Vec<String> = words.iter().map(|w| normalize(w)).collect();
+ // "o.k." and "o k" become two tokens: glue them.
+ let mut i = 0;
+ while i < norm.len() {
+ let mut ok_end = None;
+ if is_ok(&norm[i]) {
+ ok_end = Some(i + 1);
+ } else if norm[i] == "o k" || (norm[i] == "o" && norm.get(i + 1).map(String::as_str) == Some("k")) {
+ ok_end = Some(if norm[i] == "o" { i + 2 } else { i + 1 });
+ }
+ if let Some(j) = ok_end {
+ // A word made of two ("ok coucou" written as one token is rare but happens).
+ if let Some(w) = norm.get(j) {
+ let parts: Vec<&str> = w.split(' ').collect();
+ if parts.len() == 1 && is_coucou(parts[0]) {
+ return Some(words[j + 1..].join(" ").trim_start_matches([',', '.', '!', ' ']).trim().to_string());
+ }
+ if let Some(next) = norm.get(j + 1) {
+ if is_coucou_pair(w, next) {
+ return Some(words[j + 2..].join(" ").trim_start_matches([',', '.', '!', ' ']).trim().to_string());
+ }
+ }
+ }
+ }
+ // "okcoucou" in one piece.
+ if let Some(rest) = norm[i].strip_prefix("okay").or_else(|| norm[i].strip_prefix("ok")) {
+ if is_coucou(rest.trim()) {
+ return Some(words[i + 1..].join(" ").trim().to_string());
+ }
+ }
+ i += 1;
+ }
+ None
+}
+
+/// Speech or silence, 30 ms at a time, against a noise floor that follows the
+/// room. A stretch of speech is handed out once followed by `hangover` frames
+/// of silence, or when it gets too long.
+pub struct Vad {
+ floor: f32,
+ in_speech: bool,
+ speech_frames: usize,
+ silent_frames: usize,
+ frame: Vec<f32>,
+ /// The current stretch, with a little audio from before it began.
+ current: Vec<f32>,
+ preroll: std::collections::VecDeque<f32>,
+ /// Silence that ends a stretch, in frames.
+ pub hangover: usize,
+}
+
+pub const FRAME: usize = 480; // 30 ms at 16 kHz
+const PREROLL: usize = 16_000 * 3 / 10;
+const MAX_SEGMENT: usize = 16_000 * 12;
+
+impl Default for Vad {
+ fn default() -> Self {
+ Self {
+ floor: 0.004,
+ in_speech: false,
+ speech_frames: 0,
+ silent_frames: 0,
+ frame: Vec::with_capacity(FRAME),
+ current: Vec::new(),
+ preroll: std::collections::VecDeque::with_capacity(PREROLL),
+ hangover: 25,
+ }
+ }
+}
+
+impl Vad {
+ #[cfg(test)]
+ pub fn in_speech(&self) -> bool {
+ self.in_speech
+ }
+
+ /// Feeds 16 kHz samples; the stretches of speech that ended.
+ pub fn feed(&mut self, samples: &[f32]) -> Vec<Vec<f32>> {
+ let mut done = Vec::new();
+ for &s in samples {
+ self.frame.push(s);
+ if self.frame.len() < FRAME {
+ continue;
+ }
+ let rms = (self.frame.iter().map(|x| x * x).sum::<f32>() / FRAME as f32).sqrt();
+ let speech = rms > (self.floor * 3.0).max(0.012);
+ if !speech && !self.in_speech {
+ self.floor = self.floor * 0.97 + rms * 0.03;
+ }
+ if self.in_speech {
+ self.current.extend_from_slice(&self.frame);
+ if speech {
+ self.silent_frames = 0;
+ } else {
+ self.silent_frames += 1;
+ }
+ if self.silent_frames >= self.hangover || self.current.len() >= MAX_SEGMENT {
+ done.push(std::mem::take(&mut self.current));
+ self.in_speech = false;
+ self.speech_frames = 0;
+ self.silent_frames = 0;
+ }
+ } else if speech {
+ self.speech_frames += 1;
+ if self.speech_frames >= 3 {
+ self.in_speech = true;
+ self.current = self.preroll.drain(..).collect();
+ self.current.extend_from_slice(&self.frame);
+ }
+ } else {
+ self.speech_frames = 0;
+ }
+ if !self.in_speech {
+ for &x in &self.frame {
+ if self.preroll.len() == PREROLL {
+ self.preroll.pop_front();
+ }
+ self.preroll.push_back(x);
+ }
+ }
+ self.frame.clear();
+ }
+ done
+ }
+
+ /// Forgets what is being heard (Coucou speaks, or dictation starts).
+ pub fn reset(&mut self) {
+ self.in_speech = false;
+ self.speech_frames = 0;
+ self.silent_frames = 0;
+ self.current.clear();
+ self.preroll.clear();
+ self.frame.clear();
+ }
+}
+
+/// "fr-FR" for a French answer, else None (the system's language).
+pub fn speech_lang(text: &str) -> Option<&'static str> {
+ let n = normalize(text);
+ let words: Vec<&str> = n.split(' ').collect();
+ let french = ["le", "la", "les", "est", "tu", "vous", "je", "un", "une", "des", "pour", "pas", "c", "et", "sur", "avec", "ton", "ta"];
+ let count = words.iter().filter(|w| french.contains(w)).count();
+ (count * 6 >= words.len().max(1) && count >= 1).then_some("fr-FR")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_wake_phrase_however_parakeet_writes_it() {
+ assert_eq!(after_wake("OK Coucou, play some music.").as_deref(), Some("play some music."));
+ assert_eq!(after_wake("Okay, coucou! Quelle heure est-il ?").as_deref(), Some("Quelle heure est-il ?"));
+ assert_eq!(after_wake("O.K. Cuckoo what's the weather").as_deref(), Some("what's the weather"));
+ assert_eq!(after_wake("ok cou cou ajoute Stripe").as_deref(), Some("ajoute Stripe"));
+ assert_eq!(after_wake("OK Coucou.").as_deref(), Some(""));
+ assert_eq!(after_wake("Hey coucou, ça va ?").as_deref(), Some("ça va ?"));
+ assert_eq!(after_wake("on a dit coucou à Paul"), None);
+ assert_eq!(after_wake("ok let's go"), None);
+ assert_eq!(normalize("Ça VA, Élodie ?"), "ca va elodie");
+ }
+
+ #[test]
+ fn speech_is_cut_after_a_pause() {
+ let mut vad = Vad::default();
+ let silence = vec![0.0f32; 16_000];
+ let tone: Vec<f32> = (0..16_000).map(|i| (i as f32 * 0.1).sin() * 0.3).collect();
+ assert!(vad.feed(&silence).is_empty());
+ assert!(vad.feed(&tone).is_empty());
+ assert!(vad.in_speech());
+ let out = vad.feed(&silence);
+ assert_eq!(out.len(), 1);
+ // The tone, a pre-roll before it and the hangover after it.
+ assert!(out[0].len() > 16_000 && out[0].len() <= 16_000 + 25 * FRAME + PREROLL + 2 * FRAME);
+ assert!(!vad.in_speech());
+ }
+
+ #[test]
+ fn french_answers_are_spoken_in_french() {
+ assert_eq!(speech_lang("C'est fait, j'ai ajouté Stripe pour toi."), Some("fr-FR"));
+ assert_eq!(speech_lang("Done, Stripe is in the island."), None);
+ }
+}
diff --git a/windows/src/core/flow.ts b/windows/src/core/flow.ts
index 71f047a2..6c2cb6cb 100644
--- a/windows/src/core/flow.ts
+++ b/windows/src/core/flow.ts
@@ -34,6 +34,10 @@ export interface FlowPrefs {
defaultStyle: Style;
summaryEngine: "claude" | "off";
openAtLaunch: boolean;
+ /** « OK Coucou »: listen for it and answer with Claude. */
+ voice: boolean;
+ voiceOutput: VoiceOutput;
+ voiceGender: "female" | "male";
}
export type ModelState =
@@ -161,12 +165,23 @@ export interface Announcement {
export interface PillState {
phase: "idle" | "listening" | "thinking" | "locked" | "copied" | "failed" | "pastedAgain";
edge: Edge;
- key: FlowKey | "";
+ /** "wayland" where the desktop holds the shortcut. */
+ key: FlowKey | "wayland" | "";
noteStartedAt: number | null;
announcement: Announcement | null;
enabled: boolean;
+ /** « OK Coucou » is talking with you. */
+ voice: VoiceView | null;
}
+export interface VoiceView {
+ phase: "listening" | "thinking" | "speaking";
+ heard: string;
+ answer: string;
+}
+
+export type VoiceOutput = "system" | "elevenlabs" | "off";
+
export const FlowBridge = {
status: () => call<FlowStatus>("flow_status"),
setPrefs: (prefs: FlowPrefs) => call<void>("flow_set_prefs", { prefs }),
@@ -202,6 +217,10 @@ export const FlowBridge = {
pillHot: (rects: [number, number, number, number][]) => call<void>("pill_hot", { rects }),
pillHover: (inside: boolean) => call<void>("pill_hover", { inside }),
pillAction: (action: string, id?: number) => call<void>("pill_action", { action, id: id ?? null }),
+ /** The pill has said what « OK Coucou » answered. */
+ voiceDone: (id: number) => call<void>("voice_done", { id, text: null }),
+ /** speech-dispatcher or espeak-ng, where the webview has no voice. */
+ sayNative: (text: string, lang: string | null) => call<boolean>("voice_say_native", { text, lang }),
};
/** One colour per person in a call: you first, then the others. */
diff --git a/windows/src/hub/pages.ts b/windows/src/hub/pages.ts
index afbeb50b..469ca204 100644
--- a/windows/src/hub/pages.ts
+++ b/windows/src/hub/pages.ts
@@ -8,6 +8,7 @@ import { insightsPage } from "./pages/insights";
import { dictionaryPage, snippetsPage } from "./pages/words";
import { stylesPage } from "./pages/styles";
import { flowSettingsPage } from "./pages/flowsettings";
+import { voicePage } from "./pages/voice";
export const FLOW_PAGE_RENDERERS: Partial<Record<PageId, FlowPageRenderer>> = {
dictation: dictationPage,
@@ -17,4 +18,5 @@ export const FLOW_PAGE_RENDERERS: Partial<Record<PageId, FlowPageRenderer>> = {
snippets: snippetsPage,
styles: stylesPage,
flowsettings: flowSettingsPage,
+ voice: voicePage,
};
diff --git a/windows/src/hub/pages/dictation.ts b/windows/src/hub/pages/dictation.ts
index 42de6241..43932d1b 100644
--- a/windows/src/hub/pages/dictation.ts
+++ b/windows/src/hub/pages/dictation.ts
@@ -30,7 +30,7 @@ function time(ms: number): string {
export async function dictationPage(page: HTMLElement, ctx: HubContext) {
const status = ctx.status;
- const key = keyLabel(status.prefs.key, status.platform);
+ const key = keyLabel(status.keys === "wayland" ? "wayland" : status.prefs.key, status.platform);
const ready = status.prefs.enabled && status.models.parakeet.state === "ready";
const record = button(t("Record a call"), () => {
@@ -134,7 +134,7 @@ function promo(ctx: HubContext): HTMLElement {
/** Until dictation works: turn it on, download the model. */
function setup(ctx: HubContext): HTMLElement {
const status = ctx.status;
- const key = keyLabel(status.prefs.key, status.platform);
+ const key = keyLabel(status.keys === "wayland" ? "wayland" : status.prefs.key, status.platform);
const body = h("div", { style: "display:flex;flex-direction:column;gap:12px" });
body.append(
h("div", { style: "font:700 17px var(--font)", text: t("Dictate anywhere on your computer") }),
diff --git a/windows/src/hub/pages/flowsettings.ts b/windows/src/hub/pages/flowsettings.ts
index bf6be770..f195ba55 100644
--- a/windows/src/hub/pages/flowsettings.ts
+++ b/windows/src/hub/pages/flowsettings.ts
@@ -72,12 +72,14 @@ export async function flowSettingsPage(page: HTMLElement, ctx: HubContext) {
const keysCard = card(
row(t("Hold a key to dictate anywhere"), null, toggle(prefs.enabled, (on) => void ctx.setPrefs((p) => (p.enabled = on)))),
divider(),
- row(t("Hold to dictate"), prefs.key === "right_alt" ? t("On a keyboard with AltGr, the right Alt types characters: pick another key.") : null,
- select(keys, prefs.key, (key) => void ctx.setPrefs((p) => (p.key = key)).then(() => ctx.navigate("flowsettings")))),
+ status.keys === "wayland"
+ ? row(t("Hold to dictate"), null, keyKeycap("wayland", status.platform))
+ : row(t("Hold to dictate"), prefs.key === "right_alt" ? t("On a keyboard with AltGr, the right Alt types characters: pick another key.") : null,
+ select(keys, prefs.key, (key) => void ctx.setPrefs((p) => (p.key = key)).then(() => ctx.navigate("flowsettings")))),
divider(),
row(t("Where the pill is"), null, segmented(edges, prefs.edge, (edge) => void ctx.setPrefs((p) => (p.edge = edge)))),
divider(),
- row(t("Hands-free"), t("Tap twice, speak as long as you like, tap once to paste."), keyKeycap(prefs.key, status.platform, true)),
+ row(t("Hands-free"), t("Tap twice, speak as long as you like, tap once to paste."), keyKeycap(status.keys === "wayland" ? "wayland" : prefs.key, status.platform, true)),
divider(),
row(t("Paste the last dictation again"), t("When the words went to the wrong place."), button(t("Paste again"), () => void FlowBridge.pasteAgain(), "", true)),
divider(),
@@ -86,7 +88,7 @@ export async function flowSettingsPage(page: HTMLElement, ctx: HubContext) {
if (status.keys === "wayland") {
keysCard.append(h("div", {
class: "hub-warn",
- text: t("Wayland lets no app watch the keyboard: click the microphone on the pill to dictate hands-free, then ✓ to finish. Your words are copied: paste them with Ctrl+V."),
+ text: t("On Wayland, the desktop holds the dictation key for Coucou: Ctrl+Alt+V, or the keys you picked when it asked. The first paste asks once to let Coucou type for you."),
}));
} else if (status.keys !== "ok") {
keysCard.append(h("div", { class: "hub-warn", text: t("Coucou can't watch the keyboard here: use the pill's microphone to dictate.") }));
diff --git a/windows/src/hub/pages/voice.ts b/windows/src/hub/pages/voice.ts
new file mode 100644
index 00000000..c15e484a
--- /dev/null
+++ b/windows/src/hub/pages/voice.ts
@@ -0,0 +1,84 @@
+// « OK Coucou »: say it, then ask anything. Turning it on, how Coucou answers
+// out loud, and what leaves the computer. Port of the Mac's Settings → Voice.
+
+import { h, clear } from "../../views/dom";
+import { t } from "../../i18n/i18n";
+import { Bridge } from "../../core/bridge";
+import type { VoiceOutput } from "../../core/flow";
+import type { HubContext } from "../shell";
+import { button, card, divider, label, row, segmented, toggle } from "../ui";
+import { modelsControl } from "./flowsettings";
+
+const ELEVEN = "elevenlabs-api-key";
+/** Where secrets.rs keeps the keys on this OS (same words as Notch → Chat). */
+const KEY_STORE = navigator.userAgent.includes("Windows") ? "Windows Credential Manager" : "Secret Service (GNOME Keyring, KWallet)";
+
+function elevenKey(): HTMLElement {
+ const box = h("div", { style: "display:flex;align-items:center;gap:8px" });
+ const draw = async () => {
+ const present = (await Bridge.secretPresent(ELEVEN)) ?? false;
+ clear(box);
+ if (present) {
+ box.append(
+ h("span", { class: "hub-note", style: "color:var(--hub-green)", text: `✓ ${t("Key saved in the {store}.", { store: KEY_STORE })}` }),
+ button(t("Remove"), () => void Bridge.secretClear(ELEVEN).then(draw), "", true),
+ );
+ return;
+ }
+ const input = h("input", { class: "hub-input", type: "password", placeholder: t("ElevenLabs key"), autocomplete: "off", spellcheck: "false", style: "width:220px" }) as HTMLInputElement;
+ const save = button(t("Save key"), () => {
+ const value = input.value.trim();
+ if (!value) return;
+ void Bridge.secretSet(ELEVEN, value).then(draw, () => input.classList.add("err"));
+ }, "primary", true);
+ input.addEventListener("keydown", (e) => {
+ if (e.key === "Enter") save.click();
+ });
+ box.append(input, save);
+ };
+ void draw();
+ return box;
+}
+
+export async function voicePage(page: HTMLElement, ctx: HubContext) {
+ const status = await ctx.refresh();
+ const prefs = status.prefs;
+ page.append(ctx.hero("voice", "OK Coucou", t("Say « OK Coucou », then ask anything: add a pill, play music, the weather, an email to prepare.")));
+
+ const ready = status.models.parakeet.state === "ready";
+ page.append(label(t("Listening")));
+ const main = card(
+ row(t("Listen for « OK Coucou »"), t("The microphone stays open; only what follows « OK Coucou » goes further."),
+ toggle(prefs.voice, (on) => void ctx.setPrefs((p) => (p.voice = on)).then(() => ctx.navigate("voice")))),
+ divider(),
+ row(t("Parakeet, on your computer"), t("Hears « OK Coucou » here, without the internet."), modelsControl(ctx, () => ctx.navigate("voice"))),
+ );
+ if (prefs.voice && !status.hasClaudeKey) {
+ main.append(h("div", { class: "hub-warn", text: t("Coucou answers with Claude: add your Claude key in Notch → Chat.") }));
+ } else if (prefs.voice && !ready) {
+ main.append(h("div", { class: "hub-warn", text: t("Download the voice model so Coucou can hear you.") }));
+ }
+ page.append(main);
+
+ page.append(label(t("Answers")));
+ const outputs: [VoiceOutput, string][] = [["system", t("System voice")], ["elevenlabs", "ElevenLabs"], ["off", t("Text only")]];
+ const answers = card(
+ row(t("Coucou answers out loud"), null, segmented(outputs, prefs.voiceOutput, (v) => void ctx.setPrefs((p) => (p.voiceOutput = v)).then(() => ctx.navigate("voice")))),
+ );
+ if (prefs.voiceOutput === "elevenlabs") {
+ answers.append(
+ divider(),
+ row(t("Voice"), null, segmented([["female", t("Female")], ["male", t("Male")]], prefs.voiceGender, (g) => void ctx.setPrefs((p) => (p.voiceGender = g)))),
+ divider(),
+ row(t("ElevenLabs key"), null, elevenKey()),
+ );
+ }
+ page.append(answers);
+
+ page.append(label(t("Privacy")));
+ page.append(card(
+ h("div", { class: "hub-note", text: t("Your voice never leaves this computer: Parakeet turns it into text here. Only the words after « OK Coucou » go to Anthropic with your Claude key, and to ElevenLabs if you picked it for the answers.") }),
+ divider(),
+ h("div", { class: "hub-note", text: t("An email is never sent by Coucou: your mail app opens with it written, and you send it.") }),
+ ));
+}
diff --git a/windows/src/hub/shell.ts b/windows/src/hub/shell.ts
index 31a2a3b7..6e1222ce 100644
--- a/windows/src/hub/shell.ts
+++ b/windows/src/hub/shell.ts
@@ -13,7 +13,7 @@ import { icon, keyLabel } from "./ui";
export type PageId =
| "dictation" | "notes" | "insights" | "dictionary" | "snippets" | "styles" | "flowsettings"
- | "general" | "pills" | "agents" | "chat" | "integrations" | "phone" | "shortcuts";
+ | "general" | "pills" | "agents" | "chat" | "voice" | "integrations" | "phone" | "shortcuts";
interface PageDef {
id: PageId;
@@ -38,6 +38,7 @@ export const NOTCH_PAGES: PageDef[] = [
{ id: "pills", icon: "grid", title: () => t("Active pills"), subtitle: () => t("The pills shown in the island, their order and their colours.") },
{ id: "agents", icon: "terminal", title: () => t("Agents"), subtitle: () => t("Claude Code, Codex and the other agents Mochi follows.") },
{ id: "chat", icon: "chat", title: () => t("Chat"), subtitle: () => t("The AI of the island's chat, with your own keys.") },
+ { id: "voice", icon: "wave", title: () => "OK Coucou" },
{ id: "integrations", icon: "puzzle", title: () => t("Integrations"), subtitle: () => t("Stripe, GitHub, Vercel, n8n, Notion and the rest.") },
{ id: "phone", icon: "phone", title: () => t("Android phone"), subtitle: () => t("Your agents and their approvals on your Android phone, over your Wi-Fi.") },
{ id: "shortcuts", icon: "keyboard", title: () => t("Shortcuts"), subtitle: () => t("The keys that open the island and its actions.") },
@@ -71,7 +72,7 @@ export interface HubOptions {
const DEFAULT_STATUS: FlowStatus = {
prefs: {
enabled: false, key: "right_ctrl", edge: "bottom", mic: "", keepDays: 30, cleanup: false,
- defaultStyle: "natural", summaryEngine: "claude", openAtLaunch: true,
+ defaultStyle: "natural", summaryEngine: "claude", openAtLaunch: true, voice: false, voiceOutput: "system", voiceGender: "female",
},
models: { parakeet: { state: "missing" }, downloadMb: 700 },
keys: "ok",
@@ -120,12 +121,12 @@ export async function startHub(root: HTMLElement, opts: HubOptions) {
} else if (models !== "ready") {
color = "#f7b32b";
text = models === "downloading" ? t("Downloading the voice model…") : t("Download the voice model to dictate");
- } else if (status.keys === "wayland") {
+ } else if (status.keys !== "ok" && status.keys !== "wayland") {
color = "#f7b32b";
text = t("Use the pill's microphone to dictate");
} else {
color = "#34d399";
- text = t("Ready · hold {0}", { 0: keyLabel(status.prefs.key, status.platform) });
+ text = t("Ready · hold {0}", { 0: keyLabel(status.keys === "wayland" ? "wayland" : status.prefs.key, status.platform) });
}
statusLine.append(h("i", { class: "dot", style: `background:${color}` }), h("span", { text }));
}
diff --git a/windows/src/hub/ui.ts b/windows/src/hub/ui.ts
index 573e748c..07330d45 100644
--- a/windows/src/hub/ui.ts
+++ b/windows/src/hub/ui.ts
@@ -118,7 +118,7 @@ export function select<T extends string>(options: [T, string][], value: T, onCha
}
/** "Right Ctrl", "Ctrl + Win"… */
-export function keyLabel(key: FlowKey | "", platform: "windows" | "linux" = "windows"): string {
+export function keyLabel(key: FlowKey | "wayland" | "", platform: "windows" | "linux" = "windows"): string {
switch (key) {
case "right_alt":
return t("Right {0}", { 0: "Alt" });
@@ -126,13 +126,16 @@ export function keyLabel(key: FlowKey | "", platform: "windows" | "linux" = "win
return t("Right {0}", { 0: "Shift" });
case "left_ctrl_super":
return platform === "windows" ? "Ctrl + Win" : "Ctrl + Super";
+ case "wayland":
+ // Held by the desktop (GlobalShortcuts portal): the keys Coucou suggests.
+ return "Ctrl+Alt+V";
default:
return t("Right {0}", { 0: "Ctrl" });
}
}
/** The key written as keycaps: "Right Ctrl" twice for hands-free. */
-export function keyKeycap(key: FlowKey | "", platform: "windows" | "linux", twice = false): HTMLElement {
+export function keyKeycap(key: FlowKey | "wayland" | "", platform: "windows" | "linux", twice = false): HTMLElement {
const label = keyLabel(key, platform);
return keycap(twice ? `${label} ${label}` : label);
}
diff --git a/windows/src/i18n/extra.json b/windows/src/i18n/extra.json
index 790ae551..20356626 100644
--- a/windows/src/i18n/extra.json
+++ b/windows/src/i18n/extra.json
@@ -3922,17 +3922,6 @@
"ru": "Вставить снова",
"id": "Tempel lagi"
},
- "Wayland lets no app watch the keyboard: click the microphone on the pill to dictate hands-free, then ✓ to finish. Your words are copied: paste them with Ctrl+V.": {
- "zh-Hans": "Wayland 不允许任何应用监听键盘:点击胶囊上的麦克风即可免提听写,完成后点 ✓。你的文字会被复制:用 Ctrl+V 粘贴。",
- "hi": "Wayland किसी ऐप को कीबोर्ड देखने नहीं देता: हैंड्स-फ़्री बोलने के लिए पिल पर माइक्रोफ़ोन दबाएँ, फिर ख़त्म करने के लिए ✓। आपके शब्द कॉपी हो जाते हैं: उन्हें Ctrl+V से पेस्ट करें।",
- "es": "Wayland no deja que ninguna app vigile el teclado: haz clic en el micrófono de la píldora para dictar sin manos y en ✓ para terminar. Tus palabras se copian: pégalas con Ctrl+V.",
- "ar": "لا يسمح Wayland لأي تطبيق بمراقبة لوحة المفاتيح: انقر على الميكروفون في الكبسولة لتملي دون يدين، ثم على ✓ لتنهي. تُنسَخ كلماتك: الصقها بـ Ctrl+V.",
- "fr": "Wayland ne laisse aucune app surveiller le clavier : clique sur le micro de la pilule pour dicter mains libres, puis sur ✓ pour finir. Tes mots sont copiés : colle-les avec Ctrl+V.",
- "bn": "Wayland কোনো অ্যাপকে কিবোর্ড দেখতে দেয় না: হ্যান্ডস-ফ্রি ডিক্টেট করতে পিলের মাইক্রোফোনে ক্লিক করুন, শেষ করতে ✓। আপনার শব্দ কপি হয়ে যায়: Ctrl+V দিয়ে পেস্ট করুন।",
- "pt-BR": "O Wayland não deixa nenhum app observar o teclado: clique no microfone da pílula para ditar sem as mãos e em ✓ para terminar. Suas palavras são copiadas: cole com Ctrl+V.",
- "ru": "Wayland не даёт приложениям следить за клавиатурой: нажмите на микрофон на пилюле, чтобы диктовать без рук, затем ✓, чтобы закончить. Слова копируются: вставьте их через Ctrl+V.",
- "id": "Wayland tidak mengizinkan aplikasi memantau keyboard: klik mikrofon pada pil untuk mendikte tanpa tangan, lalu ✓ untuk selesai. Kata-katamu disalin: tempel dengan Ctrl+V."
- },
"Coucou can't watch the keyboard here: use the pill's microphone to dictate.": {
"zh-Hans": "Coucou 在这里无法监听键盘:请用胶囊上的麦克风听写。",
"hi": "Coucou यहाँ कीबोर्ड नहीं देख सकता: बोलकर लिखने के लिए पिल का माइक्रोफ़ोन इस्तेमाल करें।",
@@ -4207,6 +4196,204 @@
"pt-BR": "Suas palavras foram copiadas: clique onde quiser e pressione Ctrl+V.",
"ru": "Слова скопированы: щёлкните, куда их вставить, и нажмите Ctrl+V.",
"id": "Kata-katamu disalin: klik di tempat yang kamu mau lalu tekan Ctrl+V."
+ },
+ "On Wayland, the desktop holds the dictation key for Coucou: Ctrl+Alt+V, or the keys you picked when it asked. The first paste asks once to let Coucou type for you.": {
+ "zh-Hans": "在 Wayland 上,桌面会替 Coucou 管理听写按键:Ctrl+Alt+V,或者它询问时你选的按键。第一次粘贴时会询问一次,是否允许 Coucou 替你输入。",
+ "hi": "Wayland पर डेस्कटॉप Coucou के लिए डिक्टेशन कुंजी संभालता है: Ctrl+Alt+V, या वे कुंजियाँ जो आपने पूछे जाने पर चुनी थीं। पहली बार पेस्ट करते समय यह एक बार पूछता है कि क्या Coucou आपके लिए टाइप कर सकता है।",
+ "es": "En Wayland, el escritorio gestiona la tecla de dictado por Coucou: Ctrl+Alt+V, o las teclas que elegiste cuando te lo preguntó. La primera vez que pegues, te preguntará una sola vez si Coucou puede escribir por ti.",
+ "ar": "على Wayland، يتولى سطح المكتب مفتاح الإملاء نيابةً عن Coucou: ‏Ctrl+Alt+V، أو المفاتيح التي اخترتها عندما سألك. عند أول لصق يسألك مرة واحدة للسماح لـ Coucou بالكتابة نيابةً عنك.",
+ "fr": "Sous Wayland, c'est le bureau qui gère la touche de dictée pour Coucou : Ctrl+Alt+V, ou les touches que tu as choisies quand il te l'a demandé. Au premier collage, il te demande une seule fois de laisser Coucou écrire pour toi.",
+ "bn": "Wayland-এ ডেস্কটপ Coucou-র হয়ে ডিক্টেশন কী সামলায়: Ctrl+Alt+V, অথবা জিজ্ঞেস করার সময় আপনি যে কীগুলো বেছে নিয়েছিলেন। প্রথমবার পেস্ট করার সময় একবার জিজ্ঞেস করা হয় Coucou-কে আপনার হয়ে টাইপ করতে দেবেন কি না।",
+ "pt-BR": "No Wayland, a área de trabalho cuida da tecla de ditado para o Coucou: Ctrl+Alt+V, ou as teclas que você escolheu quando ela perguntou. Na primeira vez que colar, ela pergunta uma única vez se o Coucou pode digitar por você.",
+ "ru": "В Wayland клавишей диктовки для Coucou управляет рабочий стол: Ctrl+Alt+V или клавиши, которые вы выбрали, когда он спросил. При первой вставке он один раз спросит, разрешить ли Coucou печатать за вас.",
+ "id": "Di Wayland, desktop yang memegang tombol dikte untuk Coucou: Ctrl+Alt+V, atau tombol yang kamu pilih saat ditanya. Saat menempel pertama kali, desktop akan bertanya sekali apakah Coucou boleh mengetik untukmu."
+ },
+ "ElevenLabs key": {
+ "zh-Hans": "ElevenLabs 密钥",
+ "hi": "ElevenLabs कुंजी",
+ "es": "Clave de ElevenLabs",
+ "ar": "مفتاح ElevenLabs",
+ "fr": "Clé ElevenLabs",
+ "bn": "ElevenLabs কী",
+ "pt-BR": "Chave do ElevenLabs",
+ "ru": "Ключ ElevenLabs",
+ "id": "Kunci ElevenLabs"
+ },
+ "Say « OK Coucou », then ask anything: add a pill, play music, the weather, an email to prepare.": {
+ "zh-Hans": "说「OK Coucou」,然后随便问:添加一个胶囊、播放音乐、天气、准备一封邮件。",
+ "hi": "« OK Coucou » कहें, फिर कुछ भी पूछें: एक पिल जोड़ें, संगीत चलाएँ, मौसम, कोई ईमेल तैयार करें।",
+ "es": "Di « OK Coucou » y pide lo que quieras: añadir una píldora, poner música, el tiempo, un correo por preparar.",
+ "ar": "قل « OK Coucou »، ثم اطلب أي شيء: إضافة كبسولة، تشغيل الموسيقى، الطقس، بريد إلكتروني لتجهيزه.",
+ "fr": "Dis « OK Coucou », puis demande ce que tu veux : ajouter une pilule, lancer de la musique, la météo, un e-mail à préparer.",
+ "bn": "« OK Coucou » বলুন, তারপর যা খুশি জিজ্ঞেস করুন: একটি পিল যোগ করা, গান চালানো, আবহাওয়া, একটি ইমেল তৈরি করা।",
+ "pt-BR": "Diga « OK Coucou » e peça o que quiser: adicionar uma pílula, tocar música, o clima, um e-mail para preparar.",
+ "ru": "Скажите « OK Coucou » и просите что угодно: добавить плашку, включить музыку, погоду, подготовить письмо.",
+ "id": "Ucapkan « OK Coucou », lalu minta apa saja: tambah pil, putar musik, cuaca, email yang perlu disiapkan."
+ },
+ "Listening": {
+ "zh-Hans": "聆听",
+ "hi": "सुनना",
+ "es": "Escucha",
+ "ar": "الاستماع",
+ "fr": "Écoute",
+ "bn": "শোনা",
+ "pt-BR": "Escuta",
+ "ru": "Прослушивание",
+ "id": "Mendengarkan"
+ },
+ "Listen for « OK Coucou »": {
+ "zh-Hans": "聆听「OK Coucou」",
+ "hi": "« OK Coucou » के लिए सुनें",
+ "es": "Escuchar « OK Coucou »",
+ "ar": "الاستماع إلى « OK Coucou »",
+ "fr": "Écouter « OK Coucou »",
+ "bn": "« OK Coucou »-এর জন্য শুনুন",
+ "pt-BR": "Escutar « OK Coucou »",
+ "ru": "Слушать « OK Coucou »",
+ "id": "Dengarkan « OK Coucou »"
+ },
+ "The microphone stays open; only what follows « OK Coucou » goes further.": {
+ "zh-Hans": "麦克风保持开启;只有「OK Coucou」之后的内容才会被进一步处理。",
+ "hi": "माइक्रोफ़ोन खुला रहता है; केवल « OK Coucou » के बाद कही गई बात ही आगे जाती है।",
+ "es": "El micrófono sigue abierto; solo lo que va después de « OK Coucou » llega más lejos.",
+ "ar": "يبقى الميكروفون مفتوحًا؛ ولا يتجاوز الحاسوب إلا ما يأتي بعد « OK Coucou ».",
+ "fr": "Le micro reste ouvert ; seul ce qui suit « OK Coucou » va plus loin.",
+ "bn": "মাইক্রোফোন খোলা থাকে; কেবল « OK Coucou »-এর পরে যা বলা হয় সেটুকুই আরও এগোয়।",
+ "pt-BR": "O microfone fica aberto; só o que vem depois de « OK Coucou » vai adiante.",
+ "ru": "Микрофон остаётся включённым; дальше уходит только то, что сказано после « OK Coucou ».",
+ "id": "Mikrofon tetap terbuka; hanya yang diucapkan setelah « OK Coucou » yang diteruskan."
+ },
+ "Hears « OK Coucou » here, without the internet.": {
+ "zh-Hans": "在这台电脑上听到「OK Coucou」,无需联网。",
+ "hi": "« OK Coucou » यहीं सुनता है, इंटरनेट के बिना।",
+ "es": "Oye « OK Coucou » aquí, sin internet.",
+ "ar": "يسمع « OK Coucou » هنا، دون إنترنت.",
+ "fr": "Entend « OK Coucou » ici, sans internet.",
+ "bn": "« OK Coucou » এখানেই শোনে, ইন্টারনেট ছাড়াই।",
+ "pt-BR": "Ouve « OK Coucou » aqui, sem internet.",
+ "ru": "Слышит « OK Coucou » прямо здесь, без интернета.",
+ "id": "Mendengar « OK Coucou » di sini, tanpa internet."
+ },
+ "Coucou answers with Claude: add your Claude key in Notch → Chat.": {
+ "zh-Hans": "Coucou 用 Claude 回答:请在 Notch → 聊天 中添加你的 Claude 密钥。",
+ "hi": "Coucou Claude से जवाब देता है: Notch → चैट में अपनी Claude कुंजी जोड़ें।",
+ "es": "Coucou responde con Claude: añade tu clave de Claude en Notch → Chat.",
+ "ar": "يجيب Coucou باستخدام Claude: أضف مفتاح Claude الخاص بك في Notch ← الدردشة.",
+ "fr": "Coucou répond avec Claude : ajoute ta clé Claude dans Notch → Chat.",
+ "bn": "Coucou Claude দিয়ে উত্তর দেয়: Notch → চ্যাট-এ আপনার Claude কী যোগ করুন।",
+ "pt-BR": "O Coucou responde com o Claude: adicione sua chave do Claude em Notch → Chat.",
+ "ru": "Coucou отвечает с помощью Claude: добавьте свой ключ Claude в Notch → Чат.",
+ "id": "Coucou menjawab dengan Claude: tambahkan kunci Claude-mu di Notch → Obrolan."
+ },
+ "Download the voice model so Coucou can hear you.": {
+ "zh-Hans": "下载语音模型,Coucou 才能听到你。",
+ "hi": "आवाज़ मॉडल डाउनलोड करें ताकि Coucou आपको सुन सके।",
+ "es": "Descarga el modelo de voz para que Coucou pueda oírte.",
+ "ar": "نزّل نموذج الصوت حتى يتمكن Coucou من سماعك.",
+ "fr": "Télécharge le modèle vocal pour que Coucou puisse t'entendre.",
+ "bn": "ভয়েস মডেল ডাউনলোড করুন যাতে Coucou আপনাকে শুনতে পায়।",
+ "pt-BR": "Baixe o modelo de voz para o Coucou poder ouvir você.",
+ "ru": "Скачайте голосовую модель, чтобы Coucou мог вас слышать.",
+ "id": "Unduh model suara agar Coucou bisa mendengarmu."
+ },
+ "Answers": {
+ "zh-Hans": "回答",
+ "hi": "जवाब",
+ "es": "Respuestas",
+ "ar": "الإجابات",
+ "fr": "Réponses",
+ "bn": "উত্তর",
+ "pt-BR": "Respostas",
+ "ru": "Ответы",
+ "id": "Jawaban"
+ },
+ "System voice": {
+ "zh-Hans": "系统语音",
+ "hi": "सिस्टम की आवाज़",
+ "es": "Voz del sistema",
+ "ar": "صوت النظام",
+ "fr": "Voix du système",
+ "bn": "সিস্টেমের কণ্ঠ",
+ "pt-BR": "Voz do sistema",
+ "ru": "Системный голос",
+ "id": "Suara sistem"
+ },
+ "Text only": {
+ "zh-Hans": "仅文字",
+ "hi": "केवल टेक्स्ट",
+ "es": "Solo texto",
+ "ar": "نص فقط",
+ "fr": "Texte seul",
+ "bn": "শুধু টেক্সট",
+ "pt-BR": "Só texto",
+ "ru": "Только текст",
+ "id": "Teks saja"
+ },
+ "Coucou answers out loud": {
+ "zh-Hans": "Coucou 大声回答",
+ "hi": "Coucou ज़ोर से जवाब देता है",
+ "es": "Coucou responde en voz alta",
+ "ar": "يجيب Coucou بصوت عالٍ",
+ "fr": "Coucou répond à voix haute",
+ "bn": "Coucou জোরে উত্তর দেয়",
+ "pt-BR": "O Coucou responde em voz alta",
+ "ru": "Coucou отвечает вслух",
+ "id": "Coucou menjawab dengan suara"
+ },
+ "Female": {
+ "zh-Hans": "女声",
+ "hi": "महिला",
+ "es": "Femenina",
+ "ar": "أنثى",
+ "fr": "Féminine",
+ "bn": "নারী",
+ "pt-BR": "Feminina",
+ "ru": "Женский",
+ "id": "Perempuan"
+ },
+ "Male": {
+ "zh-Hans": "男声",
+ "hi": "पुरुष",
+ "es": "Masculina",
+ "ar": "ذكر",
+ "fr": "Masculine",
+ "bn": "পুরুষ",
+ "pt-BR": "Masculina",
+ "ru": "Мужской",
+ "id": "Laki-laki"
+ },
+ "Your voice never leaves this computer: Parakeet turns it into text here. Only the words after « OK Coucou » go to Anthropic with your Claude key, and to ElevenLabs if you picked it for the answers.": {
+ "zh-Hans": "你的声音永远不会离开这台电脑:Parakeet 在本机把它转成文字。只有「OK Coucou」之后的文字会用你的 Claude 密钥发送给 Anthropic;如果你选了 ElevenLabs 来朗读回答,也会发送给 ElevenLabs。",
+ "hi": "आपकी आवाज़ कभी इस कंप्यूटर से बाहर नहीं जाती: Parakeet उसे यहीं टेक्स्ट में बदलता है। केवल « OK Coucou » के बाद के शब्द आपकी Claude कुंजी के साथ Anthropic को जाते हैं, और अगर आपने जवाबों के लिए ElevenLabs चुना है तो ElevenLabs को भी।",
+ "es": "Tu voz nunca sale de este ordenador: Parakeet la convierte en texto aquí. Solo las palabras después de « OK Coucou » van a Anthropic con tu clave de Claude, y a ElevenLabs si lo elegiste para las respuestas.",
+ "ar": "لا يغادر صوتك هذا الحاسوب أبدًا: يحوّله Parakeet إلى نص هنا. لا تذهب إلا الكلمات التي تلي « OK Coucou » إلى Anthropic بمفتاح Claude الخاص بك، وإلى ElevenLabs إن اخترته للإجابات.",
+ "fr": "Ta voix ne quitte jamais cet ordinateur : Parakeet la transforme en texte ici. Seuls les mots qui suivent « OK Coucou » partent chez Anthropic avec ta clé Claude, et chez ElevenLabs si tu l'as choisi pour les réponses.",
+ "bn": "আপনার কণ্ঠ কখনো এই কম্পিউটার ছেড়ে যায় না: Parakeet এখানেই একে টেক্সটে রূপান্তর করে। কেবল « OK Coucou »-এর পরের শব্দগুলো আপনার Claude কী দিয়ে Anthropic-এ যায়, আর উত্তরের জন্য ElevenLabs বেছে নিলে ElevenLabs-এও।",
+ "pt-BR": "Sua voz nunca sai deste computador: o Parakeet a transforma em texto aqui. Só as palavras depois de « OK Coucou » vão para a Anthropic com sua chave do Claude, e para a ElevenLabs se você a escolheu para as respostas.",
+ "ru": "Ваш голос никогда не покидает этот компьютер: Parakeet превращает его в текст здесь. Только слова после « OK Coucou » уходят в Anthropic с вашим ключом Claude и в ElevenLabs, если вы выбрали его для ответов.",
+ "id": "Suaramu tidak pernah keluar dari komputer ini: Parakeet mengubahnya menjadi teks di sini. Hanya kata-kata setelah « OK Coucou » yang dikirim ke Anthropic dengan kunci Claude-mu, dan ke ElevenLabs jika kamu memilihnya untuk jawaban."
+ },
+ "An email is never sent by Coucou: your mail app opens with it written, and you send it.": {
+ "zh-Hans": "Coucou 从不发送邮件:你的邮件应用会打开并填好内容,由你来发送。",
+ "hi": "Coucou कभी ईमेल नहीं भेजता: आपका मेल ऐप उसे लिखा हुआ खोलता है, और भेजते आप हैं।",
+ "es": "Coucou nunca envía un correo: tu app de correo se abre con él escrito, y lo envías tú.",
+ "ar": "لا يرسل Coucou أي بريد إلكتروني أبدًا: يُفتح تطبيق البريد لديك والرسالة مكتوبة، وأنت من يرسلها.",
+ "fr": "Coucou n'envoie jamais d'e-mail : ton app de mail s'ouvre avec le message écrit, et c'est toi qui l'envoies.",
+ "bn": "Coucou কখনো ইমেল পাঠায় না: আপনার মেল অ্যাপ লেখা ইমেলসহ খোলে, আর পাঠান আপনি।",
+ "pt-BR": "O Coucou nunca envia um e-mail: seu app de e-mail abre com ele escrito, e quem envia é você.",
+ "ru": "Coucou никогда не отправляет письма: ваше почтовое приложение откроется с готовым текстом, а отправите его вы.",
+ "id": "Coucou tidak pernah mengirim email: aplikasi email-mu terbuka dengan email yang sudah ditulis, dan kamu yang mengirimnya."
+ },
+ "Coucou couldn't answer: check your Claude key in Notch → Chat.": {
+ "zh-Hans": "Coucou 无法回答:请检查 Notch → 聊天 中的 Claude 密钥。",
+ "hi": "Coucou जवाब नहीं दे सका: Notch → चैट में अपनी Claude कुंजी जाँचें।",
+ "es": "Coucou no pudo responder: revisa tu clave de Claude en Notch → Chat.",
+ "ar": "تعذّر على Coucou الإجابة: تحقّق من مفتاح Claude في Notch ← الدردشة.",
+ "fr": "Coucou n'a pas pu répondre : vérifie ta clé Claude dans Notch → Chat.",
+ "bn": "Coucou উত্তর দিতে পারেনি: Notch → চ্যাট-এ আপনার Claude কী যাচাই করুন।",
+ "pt-BR": "O Coucou não conseguiu responder: verifique sua chave do Claude em Notch → Chat.",
+ "ru": "Coucou не смог ответить: проверьте ключ Claude в Notch → Чат.",
+ "id": "Coucou tidak bisa menjawab: periksa kunci Claude-mu di Notch → Obrolan."
}
}
}
diff --git a/windows/src/island/voice.ts b/windows/src/island/voice.ts
new file mode 100644
index 00000000..cb8b59f1
--- /dev/null
+++ b/windows/src/island/voice.ts
@@ -0,0 +1,123 @@
+// « OK Coucou » asks the island what only it knows (src-tauri/src/voice): the
+// pills, to add, remove or pick the main one, and the user's own data for
+// Claude to answer from. Port of the Mac's VoiceTools pills and service_info.
+
+import { invoke } from "@tauri-apps/api/core";
+import { Bridge, IS_TAURI, onEvent } from "../core/bridge";
+import { MAX_DECLARED, availablePills, chooseMainPill, pillDefinition, sanitizeDeclared, toggleDeclared } from "../core/pills";
+import { Spotify } from "../core/spotify";
+import { State } from "../core/state";
+import { serviceCards } from "./phone-services";
+
+interface Ask {
+ id: number;
+ kind: "pills" | "info";
+ input: { action?: string; names?: string[] };
+}
+
+const norm = (s: string) =>
+ s.normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]/g, "");
+
+/** The pill a spoken name means: exact, then the start, then anywhere in it. */
+export function findPill(name: string): string | null {
+ const wanted = norm(name);
+ if (!wanted) return null;
+ const pills = availablePills(State.os).filter((p) => p.support !== "soon");
+ const hit =
+ pills.find((p) => norm(p.name) === wanted || norm(p.id.replace(/^(integration|agent)_/, "")) === wanted) ??
+ pills.find((p) => norm(p.name).startsWith(wanted)) ??
+ pills.find((p) => norm(p.name).includes(wanted) || wanted.includes(norm(p.name)));
+ return hit?.id ?? null;
+}
+
+function describe(): string {
+ const d = sanitizeDeclared(State.settings, State.os);
+ const name = (id: string) => pillDefinition(id)?.name ?? id;
+ const all = availablePills(State.os).filter((p) => p.support !== "soon").map((p) => p.name);
+ return JSON.stringify({
+ main: name(d.mainPill),
+ shown: d.activeIntegrations.map(name),
+ maxBesidesMain: MAX_DECLARED,
+ available: all,
+ });
+}
+
+function pills(input: Ask["input"]): string {
+ const action = input.action ?? "list";
+ if (action === "list") return describe();
+ const names = input.names ?? [];
+ const done: string[] = [];
+ const failed: string[] = [];
+ for (const spoken of names) {
+ const id = findPill(spoken);
+ if (!id) {
+ failed.push(`${spoken}: unknown pill`);
+ continue;
+ }
+ const d = sanitizeDeclared(State.settings, State.os);
+ const label = pillDefinition(id)?.name ?? id;
+ if (action === "set_main") {
+ const next = chooseMainPill(d, id, State.os);
+ if (!next) {
+ failed.push(`${label}: cannot be the main pill (only workspace tools can)`);
+ continue;
+ }
+ State.settings.mainPill = next.mainPill;
+ State.settings.activeIntegrations = next.activeIntegrations;
+ done.push(label);
+ continue;
+ }
+ const shown = id === d.mainPill || d.activeIntegrations.includes(id);
+ if ((action === "add" && shown) || (action === "remove" && !shown)) {
+ done.push(label);
+ continue;
+ }
+ const next = toggleDeclared(d, id, State.os);
+ if (!next) {
+ failed.push(action === "add" ? `${label}: already ${MAX_DECLARED} pills, remove one first` : `${label}: the main pill cannot be removed`);
+ continue;
+ }
+ State.settings.activeIntegrations = next;
+ done.push(label);
+ }
+ if (done.length) {
+ const previousMain = State.mainPillId;
+ State.loadIntegrationTasks();
+ if (State.mainPillId !== previousMain) State.setFocus(State.mainPillId);
+ void Bridge.saveSettings(State.settings);
+ }
+ return JSON.stringify({ done, failed, now: JSON.parse(describe()) });
+}
+
+function info(): string {
+ const sessions = State.tasks
+ .filter((t) => !t.isIntegration || t.state !== "idle" || t.steps.length > 0)
+ .map((t) => ({ pill: t.name, state: t.state, doing: t.steps[t.stepIndex] ?? t.steps[t.steps.length - 1] ?? null }));
+ const plan = (p: typeof State.planUsage) =>
+ p && {
+ fiveHourPercent: p.fiveHour?.usedPct ?? null,
+ weekPercent: p.sevenDay?.usedPct ?? null,
+ };
+ const s = Spotify.state;
+ return JSON.stringify({
+ pills: JSON.parse(describe()),
+ sessions,
+ claudePlan: plan(State.planUsage),
+ codexPlan: plan(State.codexPlanUsage),
+ spotify: s.running && s.track ? { playing: s.playing, title: s.track.title, artist: s.track.artist } : null,
+ services: serviceCards().map((c) => ({ service: c.title, headline: c.headline, about: c.reason, items: c.items })),
+ });
+}
+
+export function registerVoiceHandlers() {
+ if (!IS_TAURI) return;
+ void onEvent<Ask>("voice-ask", (ask) => {
+ let text: string;
+ try {
+ text = ask.kind === "pills" ? pills(ask.input ?? {}) : info();
+ } catch (err) {
+ text = `error: ${String(err)}`;
+ }
+ void invoke("voice_done", { id: ask.id, text }).catch(() => {});
+ });
+}
diff --git a/windows/src/main.ts b/windows/src/main.ts
index ec53848a..a2db5b28 100644
--- a/windows/src/main.ts
+++ b/windows/src/main.ts
@@ -9,6 +9,7 @@ import { registerHookHandlers } from "./island/hooks";
import { registerPhoneLink } from "./island/phone-link";
import { registerIntegrationHandlers, refreshConfigured } from "./island/integrations";
import { registerShortcutHandlers } from "./island/shortcuts";
+import { registerVoiceHandlers } from "./island/voice";
import { applySpotify, registerSpotifyHandlers } from "./island/spotify";
import { SPOTIFY_ID } from "./core/spotify";
import { Recap } from "./recap/recap";
@@ -138,6 +139,7 @@ async function main() {
registerPhoneLink(island);
registerIntegrationHandlers(island);
registerShortcutHandlers(island, () => setPaused(false));
+ registerVoiceHandlers();
registerSpotifyHandlers(island);
// Rust may have read Spotify before this page listened: ask once.
if (State.settings.activeIntegrations.includes(SPOTIFY_ID)) {
diff --git a/windows/src/pill/main.ts b/windows/src/pill/main.ts
index b47d8c12..7d81770a 100644
--- a/windows/src/pill/main.ts
+++ b/windows/src/pill/main.ts
@@ -18,7 +18,7 @@ const RESTING_THICKNESS = 8;
const OPEN_THICKNESS = 34;
const BANDS = 9;
-let state: PillState = { phase: "idle", edge: "bottom", key: "right_ctrl", noteStartedAt: null, announcement: null, enabled: false };
+let state: PillState = { phase: "idle", edge: "bottom", key: "right_ctrl", noteStartedAt: null, announcement: null, enabled: false, voice: null };
let platform: "windows" | "linux" = navigator.userAgent.includes("Windows") ? "windows" : "linux";
let hovered = false;
let tooltip: string | null = null;
@@ -61,7 +61,7 @@ function mochiSlot(): HTMLElement {
// ── Looks ─────────────────────────────────────────────────────────────────────
-type Look = "resting" | "toolbar" | "note" | "live" | "handsFree" | "message";
+type Look = "resting" | "toolbar" | "note" | "live" | "handsFree" | "message" | "voice";
function look(): Look {
switch (state.phase) {
@@ -75,6 +75,7 @@ function look(): Look {
case "pastedAgain":
return "message";
default:
+ if (state.voice) return "voice";
if (state.noteStartedAt) return "note";
return hovered ? "toolbar" : "resting";
}
@@ -112,6 +113,8 @@ function openLength(l: Look): number {
return 158;
case "message":
return vertical() ? 76 : measure(messageText() ?? "") + 22 + 9 + 26 + 2;
+ case "voice":
+ return 118;
default:
return 118;
}
@@ -219,6 +222,14 @@ function bubble(): HTMLElement | null {
b.append(close);
return b;
}
+ const v = state.voice;
+ if (v && (v.heard || v.answer || v.phase === "listening")) {
+ const b = el("div", "bubble voice");
+ if (v.heard) b.append(el("span", "heard", v.heard));
+ if (v.answer) b.append(el("span", "answer", v.answer));
+ else if (!v.heard) b.append(el("span", "heard", t("Listening…")));
+ return b;
+ }
const text = tooltip ?? (vertical() ? messageText() : null);
return text ? el("div", "bubble", text) : null;
}
@@ -234,7 +245,7 @@ function render() {
const length = open ? openLength(l) : RESTING_LENGTH;
const thickness = open ? OPEN_THICKNESS : RESTING_THICKNESS;
root.className = `${state.edge}${vertical() ? " vertical" : ""}`;
- const key = `${l}|${state.edge}|${state.phase}|${state.key}|${state.noteStartedAt}|${platform}`;
+ const key = `${l}|${state.edge}|${state.phase}|${state.key}|${state.noteStartedAt}|${platform}|${state.voice?.phase ?? ""}`;
if (!capsule || key !== lastKey) {
lastKey = key;
const next = el("div", "capsule");
@@ -258,7 +269,7 @@ function render() {
for (const old of root.querySelectorAll(".bubble")) old.remove();
const b = bubble();
root.replaceChildren(...(b ? [b] : []), capsule);
- mochi.setState(state.phase === "thinking" ? "thinking" : "idle");
+ mochi.setState(state.phase === "thinking" || state.voice?.phase === "thinking" ? "thinking" : "idle");
wake();
// Where the mouse is taken, once the sizes have settled.
setTimeout(reportHot, 60);
@@ -336,6 +347,22 @@ function animating(): boolean {
return visible && look() !== "resting";
}
+/** Where dot `i` goes: the voice, a slow wave while thinking, a lively one while Coucou speaks. */
+function dotGoal(now: number, i: number): number {
+ if (state.phase === "thinking") return 0.25 + 0.25 * Math.sin(now / 160 - i * 0.7);
+ if (state.phase === "listening" || state.phase === "locked") return target[i];
+ switch (state.voice?.phase) {
+ case "thinking":
+ return 0.25 + 0.25 * Math.sin(now / 160 - i * 0.7);
+ case "speaking":
+ return speaking ? 0.2 + 0.35 * Math.abs(Math.sin(now / 110 + i * 1.3) * Math.sin(now / 270 - i * 0.6)) : 0.05;
+ case "listening":
+ return 0.08 + 0.08 * Math.sin(now / 400 - i * 0.5);
+ default:
+ return 0;
+ }
+}
+
function tick(now: number) {
frame = null;
const dt = Math.min(0.05, (now - last) / 1000);
@@ -343,7 +370,7 @@ function tick(now: number) {
const l = look();
// The dots: follow the voice, or a slow wave while thinking.
for (let i = 0; i < BANDS; i++) {
- const goal = state.phase === "thinking" ? 0.25 + 0.25 * Math.sin(now / 160 - i * 0.7) : state.phase === "listening" || state.phase === "locked" ? target[i] : 0;
+ const goal = dotGoal(now, i);
levels[i] += (goal - levels[i]) * Math.min(1, dt * 14);
const bar = barsEl.children[i] as HTMLElement;
const size = 3.4 + 16 * Math.max(0, levels[i]);
@@ -384,6 +411,80 @@ function wake() {
}
}
+// ── « OK Coucou » speaks ──────────────────────────────────────────────────────
+
+interface Say {
+ id: number;
+ text: string;
+ lang: string | null;
+ /** ElevenLabs' MP3, base64. */
+ audio: string | null;
+ gender: "female" | "male";
+}
+
+let speaking = false;
+
+/** The system's voice in the answer's language, a woman's or a man's when the system says. */
+function systemVoice(lang: string | null, gender: string): SpeechSynthesisVoice | null {
+ const voices = window.speechSynthesis?.getVoices() ?? [];
+ const want = (lang ?? navigator.language).toLowerCase();
+ const short = want.split("-")[0];
+ const inLang = voices.filter((v) => v.lang.toLowerCase() === want).concat(voices.filter((v) => v.lang.toLowerCase().split(/[-_]/)[0] === short && v.lang.toLowerCase() !== want));
+ const female = /(female|woman|zira|hortense|julie|denise|aria|jenny|samantha|amelie|eloise)/i;
+ const male = /(male|man\b|david|paul|guy|henri|claude|mark|thomas)/i;
+ const pick = inLang.find((v) => (gender === "male" ? male.test(v.name) && !female.test(v.name) : female.test(v.name)));
+ return pick ?? inLang[0] ?? null;
+}
+
+async function sayWithSystem(text: string, lang: string | null, gender: string): Promise<void> {
+ const synth = window.speechSynthesis;
+ if (synth && synth.getVoices().length === 0) {
+ await new Promise<void>((r) => {
+ synth.addEventListener("voiceschanged", () => r(), { once: true });
+ setTimeout(r, 600);
+ });
+ }
+ if (synth && synth.getVoices().length > 0) {
+ await new Promise<void>((resolve) => {
+ const u = new SpeechSynthesisUtterance(text);
+ const voice = systemVoice(lang, gender);
+ if (voice) u.voice = voice;
+ u.lang = voice?.lang ?? lang ?? navigator.language;
+ u.onend = () => resolve();
+ u.onerror = () => resolve();
+ synth.cancel();
+ synth.speak(u);
+ });
+ return;
+ }
+ await FlowBridge.sayNative(text, lang);
+}
+
+async function sayMp3(base64: string): Promise<boolean> {
+ try {
+ const audio = new Audio(`data:audio/mpeg;base64,${base64}`);
+ await audio.play();
+ await new Promise<void>((resolve) => {
+ audio.onended = () => resolve();
+ audio.onerror = () => resolve();
+ });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function say(s: Say) {
+ speaking = true;
+ wake();
+ try {
+ if (!(s.audio && (await sayMp3(s.audio)))) await sayWithSystem(s.text, s.lang, s.gender);
+ } finally {
+ speaking = false;
+ void FlowBridge.voiceDone(s.id);
+ }
+}
+
// ── Boot ──────────────────────────────────────────────────────────────────────
async function main() {
@@ -392,7 +493,7 @@ async function main() {
const status = await FlowBridge.status();
if (status) platform = status.platform;
state = (await FlowBridge.pillState()) ?? state;
- visible = state.enabled || state.noteStartedAt !== null || state.announcement !== null;
+ visible = state.enabled || state.noteStartedAt !== null || state.announcement !== null || state.voice !== null;
render();
void onEvent<PillState>("pill-state", (s) => {
state = s;
@@ -415,6 +516,7 @@ async function main() {
void onEvent<boolean>("pill-pointer", (inside) => {
if (!inside) setHovered(false);
});
+ void onEvent<Say>("voice-say", (s) => void say(s));
void onEvent<Settings>("settings-changed", (s) => {
setLanguage(resolveLanguage(s.language, systemLanguages()));
render();
diff --git a/windows/src/pill/pill.css b/windows/src/pill/pill.css
index f2d7a750..96029914 100644
--- a/windows/src/pill/pill.css
+++ b/windows/src/pill/pill.css
@@ -256,6 +256,37 @@ body {
align-items: center;
}
+.bubble.voice {
+ border-radius: 16px;
+ height: auto;
+ padding: 9px 14px;
+ flex-direction: column;
+ align-items: flex-start;
+ gap: 3px;
+ white-space: normal;
+ max-width: 440px;
+}
+
+.bubble.voice .heard {
+ color: #9a9ea8;
+ font-weight: 500;
+ line-height: 1.3;
+ display: -webkit-box;
+ -webkit-line-clamp: 2;
+ -webkit-box-orient: vertical;
+ overflow: hidden;
+}
+
+.bubble.voice .answer {
+ color: #f2f3f5;
+ font-weight: 500;
+ line-height: 1.35;
+ display: -webkit-box;
+ -webkit-line-clamp: 4;
+ -webkit-box-orient: vertical;
+ overflow: hidden;
+}
+
.bubble .icon {
width: 18px;
height: 18px;