Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(voice): download the optional local model from Connectors

Alan Buscaglia4848d24f7f8c
diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx
index 691288e..869594e 100644
--- a/packages/ui/src/App.tsx
+++ b/packages/ui/src/App.tsx
@@ -172,6 +172,7 @@ function Assistant({ surface }: { surface: Surface }) {
focusKey={focusKey}
{...(surface === "panel" ? { onHide: hidePanel } : {})}
{...(computer ? { computer } : {})}
+ {...(desktop && surface === "panel" ? { voiceModel: true } : {})}
{...(desktop && surface === "panel" ? { panelWindow: panel.controls } : {})}
{...(upload ? { upload } : {})}
/>
diff --git a/packages/ui/src/components/ChatSurface.tsx b/packages/ui/src/components/ChatSurface.tsx
index 6a22208..2823ff0 100644
--- a/packages/ui/src/components/ChatSurface.tsx
+++ b/packages/ui/src/components/ChatSurface.tsx
@@ -43,6 +43,8 @@ interface ChatSurfaceProps {
openUrl?: (url: string) => void;
/** Computer control, in the desktop panel only. */
computer?: ComputerControls;
+ /** The desktop panel offers the voice engine and its optional local model (S30.5). */
+ voiceModel?: boolean;
/** Hiding the rose and full screen (S26), in the desktop panel only; the web page already fills the browser. */
panelWindow?: PanelWindowControls;
/** Sends files to the daemon (S31); without it the composer has no attachments. */
@@ -74,6 +76,7 @@ export function ChatSurface({
dispatch = () => {},
openUrl = openExternal,
computer,
+ voiceModel,
panelWindow,
upload,
}: ChatSurfaceProps) {
@@ -265,6 +268,7 @@ export function ChatSurface({
dispatch={dispatch}
openUrl={openUrl}
{...(computer?.available ? { computer } : {})}
+ {...(voiceModel ? { voiceModel } : {})}
/>
</main>
) : null}
diff --git a/packages/ui/src/components/ConnectorsPanel.tsx b/packages/ui/src/components/ConnectorsPanel.tsx
index 805a701..1581ce1 100644
--- a/packages/ui/src/components/ConnectorsPanel.tsx
+++ b/packages/ui/src/components/ConnectorsPanel.tsx
@@ -5,6 +5,7 @@ import "../accounts.css";
import { FlowEvent, FlowPrompt } from "./AccountsPanel.tsx";
import { type ComputerControls, ComputerEntry } from "./Computer.tsx";
import type { Send } from "./types.ts";
+import { VoiceModelEntry } from "./VoiceModel.tsx";
interface ConnectorsPanelProps {
connectors: ConnectorsState;
@@ -13,6 +14,8 @@ interface ConnectorsPanelProps {
openUrl: (url: string) => void;
/** Computer control, in the desktop app on macOS only. */
computer?: ComputerControls;
+ /** The desktop app's voice engine and optional local model (S30.5). */
+ voiceModel?: boolean;
}
const STATUS_LABELS: Record<ConnectorStatus, string> = {
@@ -28,7 +31,14 @@ export const addConnectorRequest = (what: string) =>
`Please help me add a connector for ${what}. Find its MCP server and draft it for me to review.`;
/** The apps the assistant can use, each signed in from here and set to read only or read and send. */
-export function ConnectorsPanel({ connectors, send, dispatch, openUrl, computer }: ConnectorsPanelProps) {
+export function ConnectorsPanel({
+ connectors,
+ send,
+ dispatch,
+ openUrl,
+ computer,
+ voiceModel,
+}: ConnectorsPanelProps) {
const [guideFor, setGuideFor] = useState<string>();
const [adding, setAdding] = useState(false);
const flow = connectors.flow;
@@ -138,6 +148,7 @@ export function ConnectorsPanel({ connectors, send, dispatch, openUrl, computer
{...withInfo(connectors.list?.find((c) => c.builtin && c.id === "computer"))}
/>
) : null}
+ {voiceModel ? <VoiceModelEntry /> : null}
{/* Built-in entries are the desktop app's; the web page never shows them. */}
{(connectors.list ?? [])
.filter((c) => !c.builtin)
diff --git a/packages/ui/src/components/VoiceModel.tsx b/packages/ui/src/components/VoiceModel.tsx
new file mode 100644
index 0000000..ca34103
--- /dev/null
+++ b/packages/ui/src/components/VoiceModel.tsx
@@ -0,0 +1,116 @@
+import { useCallback, useEffect, useState } from "react";
+import {
+ cancelVoiceModel,
+ downloadVoiceModel,
+ onVoiceModel,
+ removeVoiceModel,
+ type VoiceModelEvent,
+ type VoiceModelStatus,
+ voiceModelStatus,
+} from "../voice-native.ts";
+
+/** The archive's size (pinned in the desktop app), shown before anything downloads. */
+const DOWNLOAD_BYTES = 487_170_055;
+
+const ENGINES: Record<VoiceModelStatus["engine"], string> = {
+ parakeet: "Using: local model (Parakeet), works offline",
+ apple: "Using: macOS speech",
+ none: "Voice input needs the local model on this computer",
+};
+
+const megabytes = (bytes: number) => `${Math.round(bytes / 1_000_000)} MB`;
+
+type Progress = { kind: "downloading"; percent: number } | { kind: "verifying" };
+
+/** The Voice entry in Connectors (desktop only): which engine listens, and the optional local model. */
+export function VoiceModelEntry() {
+ const [status, setStatus] = useState<VoiceModelStatus>();
+ const [progress, setProgress] = useState<Progress>();
+ const [error, setError] = useState<string>();
+
+ const refresh = useCallback(() => {
+ voiceModelStatus().then(
+ (next) => {
+ setStatus(next);
+ if (next.downloading) {
+ const percent = next.total ? Math.floor(((next.received ?? 0) / next.total) * 100) : 0;
+ setProgress({ kind: "downloading", percent });
+ }
+ },
+ () => setStatus(undefined),
+ );
+ }, []);
+
+ useEffect(() => {
+ refresh();
+ let stop: (() => void) | undefined;
+ let closed = false;
+ const handle = (event: VoiceModelEvent) => {
+ if (event.state === "downloading") {
+ const percent = event.total ? Math.floor(((event.received ?? 0) / event.total) * 100) : 0;
+ setError(undefined);
+ setProgress({ kind: "downloading", percent });
+ } else if (event.state === "verifying") {
+ setProgress({ kind: "verifying" });
+ } else {
+ setProgress(undefined);
+ if (event.state === "failed") setError(event.message ?? "The download did not finish.");
+ refresh();
+ }
+ };
+ onVoiceModel(handle).then(
+ (unlisten) => {
+ if (closed) unlisten();
+ else stop = unlisten;
+ },
+ () => {},
+ );
+ return () => {
+ closed = true;
+ stop?.();
+ };
+ }, [refresh]);
+
+ const act = (action: () => Promise<void>) => {
+ setError(undefined);
+ action().catch((reason: unknown) => setError(typeof reason === "string" ? reason : String(reason)));
+ };
+
+ return (
+ <li className="computer-entry" aria-labelledby="connector-voice">
+ <div className="provider-name">
+ <b id="connector-voice">Voice</b>
+ <span className="chip connector-connected">Built in</span>
+ </div>
+ {status ? <p className="connector-note">{ENGINES[status.engine]}</p> : null}
+ <p className="connector-note">
+ The local model (NVIDIA Parakeet) understands 25 languages, including Spanish, and never sends your
+ voice anywhere. It is optional: nothing downloads until you ask.
+ </p>
+ {progress?.kind === "downloading" ? (
+ <div className="voice-model-progress">
+ <progress aria-label="Downloading the local voice model" max={100} value={progress.percent} />
+ <span>Downloading… {progress.percent}%</span>
+ <button type="button" onClick={() => act(cancelVoiceModel)}>
+ Cancel
+ </button>
+ </div>
+ ) : progress?.kind === "verifying" ? (
+ <p className="connector-note">Checking and unpacking…</p>
+ ) : status?.installed ? (
+ <button type="button" onClick={() => act(removeVoiceModel)}>
+ Remove local model ({megabytes(status.bytes ?? 0)})
+ </button>
+ ) : status ? (
+ <button type="button" className="primary" onClick={() => act(downloadVoiceModel)}>
+ Download local voice model ({megabytes(DOWNLOAD_BYTES)})
+ </button>
+ ) : null}
+ {error ? (
+ <p className="connector-note voice-model-error" role="alert">
+ {error}
+ </p>
+ ) : null}
+ </li>
+ );
+}
diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.css
index ecd6d9f..72d36ab 100644
--- a/packages/ui/src/styles.css
+++ b/packages/ui/src/styles.css
@@ -1796,3 +1796,20 @@ html[data-surface="glow"] body {
.chat-fullscreen .model-picker {
margin-inline: max(12px, calc((100% - var(--reading-width)) / 2));
}
+
+/* The local voice model's download (S30.5). */
+.voice-model-progress {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ margin-top: 6px;
+}
+
+.voice-model-progress progress {
+ flex: 1;
+ accent-color: var(--accent);
+}
+
+.voice-model-error {
+ color: var(--red, #ff6b8a);
+}
diff --git a/packages/ui/src/voice-native.ts b/packages/ui/src/voice-native.ts
index a40c0cf..4e17c0b 100644
--- a/packages/ui/src/voice-native.ts
+++ b/packages/ui/src/voice-native.ts
@@ -71,3 +71,36 @@ export function startRefusal(error: unknown): string {
if (error instanceof Error && error.message) return error.message;
return "Could not start listening.";
}
+
+/**
+ * The optional local voice model (S30.5): `voice_model_status|download|cancel|remove` and the
+ * `voice://model` event. Nothing downloads unless the user asks.
+ */
+export interface VoiceModelStatus {
+ installed: boolean;
+ bytes?: number;
+ downloading: boolean;
+ received?: number;
+ total?: number;
+ engine: "parakeet" | "apple" | "none";
+}
+
+export interface VoiceModelEvent {
+ state: "downloading" | "verifying" | "installed" | "failed" | "removed";
+ received?: number;
+ total?: number;
+ message?: string;
+}
+
+export const voiceModelStatus = () => invoke<VoiceModelStatus>("voice_model_status");
+export const downloadVoiceModel = () => invoke<void>("voice_model_download");
+export const cancelVoiceModel = () => invoke<void>("voice_model_cancel");
+export const removeVoiceModel = () => invoke<void>("voice_model_remove");
+
+/** Listens to the model's download and install progress. */
+export async function onVoiceModel(handler: (event: VoiceModelEvent) => void): Promise<() => void> {
+ const { listen } = await import("@tauri-apps/api/event");
+ return listen<VoiceModelEvent>("voice://model", (event) => {
+ if (event.payload && typeof event.payload.state === "string") handler(event.payload);
+ });
+}
diff --git a/packages/ui/test/voice-model.test.tsx b/packages/ui/test/voice-model.test.tsx
new file mode 100644
index 0000000..5d6f307
--- /dev/null
+++ b/packages/ui/test/voice-model.test.tsx
@@ -0,0 +1,108 @@
+import { act, render, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { ConnectorsPanel } from "../src/components/ConnectorsPanel.tsx";
+import { VoiceModelEntry } from "../src/components/VoiceModel.tsx";
+
+const TOTAL = 487_170_055;
+
+const h = vi.hoisted(() => ({
+ status: {} as Record<string, unknown>,
+ invoke: vi.fn(),
+ listeners: new Map<string, (event: { payload: unknown }) => void>(),
+}));
+
+vi.mock("@tauri-apps/api/core", () => ({
+ isTauri: () => true,
+ invoke: (command: string, args?: unknown) => h.invoke(command, args),
+}));
+
+vi.mock("@tauri-apps/api/event", () => ({
+ listen: async (name: string, handler: (event: { payload: unknown }) => void) => {
+ h.listeners.set(name, handler);
+ return () => h.listeners.delete(name);
+ },
+}));
+
+const emit = (payload: unknown) =>
+ act(async () => {
+ h.listeners.get("voice://model")?.({ payload });
+ });
+
+beforeEach(() => {
+ h.status = { installed: false, downloading: false, engine: "apple" };
+ h.listeners.clear();
+ h.invoke.mockReset();
+ h.invoke.mockImplementation(async (command: string) => {
+ if (command === "voice_model_status") return h.status;
+ return undefined;
+ });
+});
+
+afterEach(() => vi.clearAllMocks());
+
+describe("local voice model (S30.5)", () => {
+ it("downloads only when asked, showing the size and the progress", async () => {
+ render(<VoiceModelEntry />);
+ expect(await screen.findByText("Using: macOS speech")).toBeInTheDocument();
+ expect(h.invoke).not.toHaveBeenCalledWith("voice_model_download", undefined);
+
+ await userEvent.click(screen.getByRole("button", { name: "Download local voice model (487 MB)" }));
+ expect(h.invoke).toHaveBeenCalledWith("voice_model_download", undefined);
+
+ await emit({ state: "downloading", received: TOTAL / 4, total: TOTAL });
+ expect(screen.getByRole("progressbar", { name: "Downloading the local voice model" })).toHaveAttribute(
+ "value",
+ "25",
+ );
+ expect(screen.getByText("Downloading… 25%")).toBeInTheDocument();
+
+ await emit({ state: "verifying" });
+ expect(screen.getByText("Checking and unpacking…")).toBeInTheDocument();
+ });
+
+ it("cancels a download", async () => {
+ h.status = { installed: false, downloading: true, received: 10, total: TOTAL, engine: "apple" };
+ render(<VoiceModelEntry />);
+ await userEvent.click(await screen.findByRole("button", { name: "Cancel" }));
+ expect(h.invoke).toHaveBeenCalledWith("voice_model_cancel", undefined);
+ h.status = { installed: false, downloading: false, engine: "apple" };
+ await emit({ state: "failed", message: "The download was cancelled." });
+ expect(screen.getByText("The download was cancelled.")).toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Download local voice model (487 MB)" })).toBeInTheDocument();
+ });
+
+ it("uses the installed model and can remove it", async () => {
+ h.status = { installed: true, bytes: 670_478_772, downloading: false, engine: "parakeet" };
+ render(<VoiceModelEntry />);
+ expect(await screen.findByText("Using: local model (Parakeet), works offline")).toBeInTheDocument();
+
+ h.status = { installed: false, downloading: false, engine: "apple" };
+ await userEvent.click(screen.getByRole("button", { name: "Remove local model (670 MB)" }));
+ expect(h.invoke).toHaveBeenCalledWith("voice_model_remove", undefined);
+ await emit({ state: "removed" });
+ await waitFor(() => expect(screen.getByText("Using: macOS speech")).toBeInTheDocument());
+ });
+
+ it("shows a failed download's reason and offers the download again", async () => {
+ render(<VoiceModelEntry />);
+ await screen.findByText("Using: macOS speech");
+ await emit({ state: "failed", message: "The download did not match its checksum." });
+ expect(screen.getByRole("alert")).toHaveTextContent("The download did not match its checksum.");
+ expect(screen.getByRole("button", { name: "Download local voice model (487 MB)" })).toBeInTheDocument();
+ });
+
+ it("appears in Connectors only when the desktop app offers it", () => {
+ const props = {
+ connectors: { open: true, list: [] },
+ send: () => {},
+ dispatch: () => {},
+ openUrl: () => {},
+ };
+ const { unmount } = render(<ConnectorsPanel {...props} />);
+ expect(screen.queryByText("Voice")).toBeNull();
+ unmount();
+ render(<ConnectorsPanel {...props} voiceModel />);
+ expect(screen.getByText("Voice")).toBeInTheDocument();
+ });
+});