A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
feat(voice): download the optional local model from Connectors
Alan Buscaglia4848d24f7f8c
diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsxindex 691288e..869594e 100644--- a/packages/ui/src/App.tsx+++ b/packages/ui/src/App.tsx@@ -172,6 +172,7 @@ function Assistant({ surface }: { surface: Surface }) {focusKey={focusKey}{...(surface === "panel" ? { onHide: hidePanel } : {})}{...(computer ? { computer } : {})}+ {...(desktop && surface === "panel" ? { voiceModel: true } : {})}{...(desktop && surface === "panel" ? { panelWindow: panel.controls } : {})}{...(upload ? { upload } : {})}/>diff --git a/packages/ui/src/components/ChatSurface.tsx b/packages/ui/src/components/ChatSurface.tsxindex 6a22208..2823ff0 100644--- a/packages/ui/src/components/ChatSurface.tsx+++ b/packages/ui/src/components/ChatSurface.tsx@@ -43,6 +43,8 @@ interface ChatSurfaceProps {openUrl?: (url: string) => void;/** Computer control, in the desktop panel only. */computer?: ComputerControls;+ /** The desktop panel offers the voice engine and its optional local model (S30.5). */+ voiceModel?: boolean;/** Hiding the rose and full screen (S26), in the desktop panel only; the web page already fills the browser. */panelWindow?: PanelWindowControls;/** Sends files to the daemon (S31); without it the composer has no attachments. */@@ -74,6 +76,7 @@ export function ChatSurface({dispatch = () => {},openUrl = openExternal,computer,+ voiceModel,panelWindow,upload,}: ChatSurfaceProps) {@@ -265,6 +268,7 @@ export function ChatSurface({dispatch={dispatch}openUrl={openUrl}{...(computer?.available ? { computer } : {})}+ {...(voiceModel ? { voiceModel } : {})}/></main>) : null}diff --git a/packages/ui/src/components/ConnectorsPanel.tsx b/packages/ui/src/components/ConnectorsPanel.tsxindex 805a701..1581ce1 100644--- a/packages/ui/src/components/ConnectorsPanel.tsx+++ b/packages/ui/src/components/ConnectorsPanel.tsx@@ -5,6 +5,7 @@ import "../accounts.css";import { FlowEvent, FlowPrompt } from "./AccountsPanel.tsx";import { type ComputerControls, ComputerEntry } from "./Computer.tsx";import type { Send } from "./types.ts";+import { VoiceModelEntry } from "./VoiceModel.tsx";interface ConnectorsPanelProps {connectors: ConnectorsState;@@ -13,6 +14,8 @@ interface ConnectorsPanelProps {openUrl: (url: string) => void;/** Computer control, in the desktop app on macOS only. */computer?: ComputerControls;+ /** The desktop app's voice engine and optional local model (S30.5). */+ voiceModel?: boolean;}const STATUS_LABELS: Record<ConnectorStatus, string> = {@@ -28,7 +31,14 @@ export const addConnectorRequest = (what: string) =>`Please help me add a connector for ${what}. Find its MCP server and draft it for me to review.`;/** The apps the assistant can use, each signed in from here and set to read only or read and send. */-export function ConnectorsPanel({ connectors, send, dispatch, openUrl, computer }: ConnectorsPanelProps) {+export function ConnectorsPanel({+ connectors,+ send,+ dispatch,+ openUrl,+ computer,+ voiceModel,+}: ConnectorsPanelProps) {const [guideFor, setGuideFor] = useState<string>();const [adding, setAdding] = useState(false);const flow = connectors.flow;@@ -138,6 +148,7 @@ export function ConnectorsPanel({ connectors, send, dispatch, openUrl, computer{...withInfo(connectors.list?.find((c) => c.builtin && c.id === "computer"))}/>) : null}+ {voiceModel ? <VoiceModelEntry /> : null}{/* Built-in entries are the desktop app's; the web page never shows them. */}{(connectors.list ?? []).filter((c) => !c.builtin)diff --git a/packages/ui/src/components/VoiceModel.tsx b/packages/ui/src/components/VoiceModel.tsxnew file mode 100644index 0000000..ca34103--- /dev/null+++ b/packages/ui/src/components/VoiceModel.tsx@@ -0,0 +1,116 @@+import { useCallback, useEffect, useState } from "react";+import {+ cancelVoiceModel,+ downloadVoiceModel,+ onVoiceModel,+ removeVoiceModel,+ type VoiceModelEvent,+ type VoiceModelStatus,+ voiceModelStatus,+} from "../voice-native.ts";++/** The archive's size (pinned in the desktop app), shown before anything downloads. */+const DOWNLOAD_BYTES = 487_170_055;++const ENGINES: Record<VoiceModelStatus["engine"], string> = {+ parakeet: "Using: local model (Parakeet), works offline",+ apple: "Using: macOS speech",+ none: "Voice input needs the local model on this computer",+};++const megabytes = (bytes: number) => `${Math.round(bytes / 1_000_000)} MB`;++type Progress = { kind: "downloading"; percent: number } | { kind: "verifying" };++/** The Voice entry in Connectors (desktop only): which engine listens, and the optional local model. */+export function VoiceModelEntry() {+ const [status, setStatus] = useState<VoiceModelStatus>();+ const [progress, setProgress] = useState<Progress>();+ const [error, setError] = useState<string>();++ const refresh = useCallback(() => {+ voiceModelStatus().then(+ (next) => {+ setStatus(next);+ if (next.downloading) {+ const percent = next.total ? Math.floor(((next.received ?? 0) / next.total) * 100) : 0;+ setProgress({ kind: "downloading", percent });+ }+ },+ () => setStatus(undefined),+ );+ }, []);++ useEffect(() => {+ refresh();+ let stop: (() => void) | undefined;+ let closed = false;+ const handle = (event: VoiceModelEvent) => {+ if (event.state === "downloading") {+ const percent = event.total ? Math.floor(((event.received ?? 0) / event.total) * 100) : 0;+ setError(undefined);+ setProgress({ kind: "downloading", percent });+ } else if (event.state === "verifying") {+ setProgress({ kind: "verifying" });+ } else {+ setProgress(undefined);+ if (event.state === "failed") setError(event.message ?? "The download did not finish.");+ refresh();+ }+ };+ onVoiceModel(handle).then(+ (unlisten) => {+ if (closed) unlisten();+ else stop = unlisten;+ },+ () => {},+ );+ return () => {+ closed = true;+ stop?.();+ };+ }, [refresh]);++ const act = (action: () => Promise<void>) => {+ setError(undefined);+ action().catch((reason: unknown) => setError(typeof reason === "string" ? reason : String(reason)));+ };++ return (+ <li className="computer-entry" aria-labelledby="connector-voice">+ <div className="provider-name">+ <b id="connector-voice">Voice</b>+ <span className="chip connector-connected">Built in</span>+ </div>+ {status ? <p className="connector-note">{ENGINES[status.engine]}</p> : null}+ <p className="connector-note">+ The local model (NVIDIA Parakeet) understands 25 languages, including Spanish, and never sends your+ voice anywhere. It is optional: nothing downloads until you ask.+ </p>+ {progress?.kind === "downloading" ? (+ <div className="voice-model-progress">+ <progress aria-label="Downloading the local voice model" max={100} value={progress.percent} />+ <span>Downloading… {progress.percent}%</span>+ <button type="button" onClick={() => act(cancelVoiceModel)}>+ Cancel+ </button>+ </div>+ ) : progress?.kind === "verifying" ? (+ <p className="connector-note">Checking and unpacking…</p>+ ) : status?.installed ? (+ <button type="button" onClick={() => act(removeVoiceModel)}>+ Remove local model ({megabytes(status.bytes ?? 0)})+ </button>+ ) : status ? (+ <button type="button" className="primary" onClick={() => act(downloadVoiceModel)}>+ Download local voice model ({megabytes(DOWNLOAD_BYTES)})+ </button>+ ) : null}+ {error ? (+ <p className="connector-note voice-model-error" role="alert">+ {error}+ </p>+ ) : null}+ </li>+ );+}diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.cssindex ecd6d9f..72d36ab 100644--- a/packages/ui/src/styles.css+++ b/packages/ui/src/styles.css@@ -1796,3 +1796,20 @@ html[data-surface="glow"] body {.chat-fullscreen .model-picker {margin-inline: max(12px, calc((100% - var(--reading-width)) / 2));}++/* The local voice model's download (S30.5). */+.voice-model-progress {+ display: flex;+ align-items: center;+ gap: 8px;+ margin-top: 6px;+}++.voice-model-progress progress {+ flex: 1;+ accent-color: var(--accent);+}++.voice-model-error {+ color: var(--red, #ff6b8a);+}diff --git a/packages/ui/src/voice-native.ts b/packages/ui/src/voice-native.tsindex a40c0cf..4e17c0b 100644--- a/packages/ui/src/voice-native.ts+++ b/packages/ui/src/voice-native.ts@@ -71,3 +71,36 @@ export function startRefusal(error: unknown): string {if (error instanceof Error && error.message) return error.message;return "Could not start listening.";}++/**+ * The optional local voice model (S30.5): `voice_model_status|download|cancel|remove` and the+ * `voice://model` event. Nothing downloads unless the user asks.+ */+export interface VoiceModelStatus {+ installed: boolean;+ bytes?: number;+ downloading: boolean;+ received?: number;+ total?: number;+ engine: "parakeet" | "apple" | "none";+}++export interface VoiceModelEvent {+ state: "downloading" | "verifying" | "installed" | "failed" | "removed";+ received?: number;+ total?: number;+ message?: string;+}++export const voiceModelStatus = () => invoke<VoiceModelStatus>("voice_model_status");+export const downloadVoiceModel = () => invoke<void>("voice_model_download");+export const cancelVoiceModel = () => invoke<void>("voice_model_cancel");+export const removeVoiceModel = () => invoke<void>("voice_model_remove");++/** Listens to the model's download and install progress. */+export async function onVoiceModel(handler: (event: VoiceModelEvent) => void): Promise<() => void> {+ const { listen } = await import("@tauri-apps/api/event");+ return listen<VoiceModelEvent>("voice://model", (event) => {+ if (event.payload && typeof event.payload.state === "string") handler(event.payload);+ });+}diff --git a/packages/ui/test/voice-model.test.tsx b/packages/ui/test/voice-model.test.tsxnew file mode 100644index 0000000..5d6f307--- /dev/null+++ b/packages/ui/test/voice-model.test.tsx@@ -0,0 +1,108 @@+import { act, render, screen, waitFor } from "@testing-library/react";+import userEvent from "@testing-library/user-event";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { ConnectorsPanel } from "../src/components/ConnectorsPanel.tsx";+import { VoiceModelEntry } from "../src/components/VoiceModel.tsx";++const TOTAL = 487_170_055;++const h = vi.hoisted(() => ({+ status: {} as Record<string, unknown>,+ invoke: vi.fn(),+ listeners: new Map<string, (event: { payload: unknown }) => void>(),+}));++vi.mock("@tauri-apps/api/core", () => ({+ isTauri: () => true,+ invoke: (command: string, args?: unknown) => h.invoke(command, args),+}));++vi.mock("@tauri-apps/api/event", () => ({+ listen: async (name: string, handler: (event: { payload: unknown }) => void) => {+ h.listeners.set(name, handler);+ return () => h.listeners.delete(name);+ },+}));++const emit = (payload: unknown) =>+ act(async () => {+ h.listeners.get("voice://model")?.({ payload });+ });++beforeEach(() => {+ h.status = { installed: false, downloading: false, engine: "apple" };+ h.listeners.clear();+ h.invoke.mockReset();+ h.invoke.mockImplementation(async (command: string) => {+ if (command === "voice_model_status") return h.status;+ return undefined;+ });+});++afterEach(() => vi.clearAllMocks());++describe("local voice model (S30.5)", () => {+ it("downloads only when asked, showing the size and the progress", async () => {+ render(<VoiceModelEntry />);+ expect(await screen.findByText("Using: macOS speech")).toBeInTheDocument();+ expect(h.invoke).not.toHaveBeenCalledWith("voice_model_download", undefined);++ await userEvent.click(screen.getByRole("button", { name: "Download local voice model (487 MB)" }));+ expect(h.invoke).toHaveBeenCalledWith("voice_model_download", undefined);++ await emit({ state: "downloading", received: TOTAL / 4, total: TOTAL });+ expect(screen.getByRole("progressbar", { name: "Downloading the local voice model" })).toHaveAttribute(+ "value",+ "25",+ );+ expect(screen.getByText("Downloading… 25%")).toBeInTheDocument();++ await emit({ state: "verifying" });+ expect(screen.getByText("Checking and unpacking…")).toBeInTheDocument();+ });++ it("cancels a download", async () => {+ h.status = { installed: false, downloading: true, received: 10, total: TOTAL, engine: "apple" };+ render(<VoiceModelEntry />);+ await userEvent.click(await screen.findByRole("button", { name: "Cancel" }));+ expect(h.invoke).toHaveBeenCalledWith("voice_model_cancel", undefined);+ h.status = { installed: false, downloading: false, engine: "apple" };+ await emit({ state: "failed", message: "The download was cancelled." });+ expect(screen.getByText("The download was cancelled.")).toBeInTheDocument();+ expect(screen.getByRole("button", { name: "Download local voice model (487 MB)" })).toBeInTheDocument();+ });++ it("uses the installed model and can remove it", async () => {+ h.status = { installed: true, bytes: 670_478_772, downloading: false, engine: "parakeet" };+ render(<VoiceModelEntry />);+ expect(await screen.findByText("Using: local model (Parakeet), works offline")).toBeInTheDocument();++ h.status = { installed: false, downloading: false, engine: "apple" };+ await userEvent.click(screen.getByRole("button", { name: "Remove local model (670 MB)" }));+ expect(h.invoke).toHaveBeenCalledWith("voice_model_remove", undefined);+ await emit({ state: "removed" });+ await waitFor(() => expect(screen.getByText("Using: macOS speech")).toBeInTheDocument());+ });++ it("shows a failed download's reason and offers the download again", async () => {+ render(<VoiceModelEntry />);+ await screen.findByText("Using: macOS speech");+ await emit({ state: "failed", message: "The download did not match its checksum." });+ expect(screen.getByRole("alert")).toHaveTextContent("The download did not match its checksum.");+ expect(screen.getByRole("button", { name: "Download local voice model (487 MB)" })).toBeInTheDocument();+ });++ it("appears in Connectors only when the desktop app offers it", () => {+ const props = {+ connectors: { open: true, list: [] },+ send: () => {},+ dispatch: () => {},+ openUrl: () => {},+ };+ const { unmount } = render(<ConnectorsPanel {...props} />);+ expect(screen.queryByText("Voice")).toBeNull();+ unmount();+ render(<ConnectorsPanel {...props} voiceModel />);+ expect(screen.getByText("Voice")).toBeInTheDocument();+ });+});