Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(assistant): BYOK-only vision override and declared cloud reasoning purpose (#2115)
Gabriel Steinbb881d10b79c
diff --git a/src/components/settings/DictationAgentSettings.tsx b/src/components/settings/DictationAgentSettings.tsxindex 601171a80..9573ecff5 100644--- a/src/components/settings/DictationAgentSettings.tsx+++ b/src/components/settings/DictationAgentSettings.tsx@@ -2,7 +2,11 @@ import { useCallback, useState } from "react";import { useTranslation } from "react-i18next";import { Monitor } from "lucide-react";import { useSettingsStore } from "../../stores/settingsStore";-import { isAgentAllowed, isScreenContextAllowed } from "../../stores/policyRules";+import {+ isAgentAllowed,+ isModeAllowedByPolicy,+ isScreenContextAllowed,+} from "../../stores/policyRules";import { usePolicyStore } from "../../stores/policyStore";import { useAgentName } from "../../utils/agentName";import { useDialogs } from "../../hooks/useDialogs";@@ -34,6 +38,9 @@ export default function DictationAgentSettings() {} = useScreenRecordingPermission();const agentAllowed = usePolicyStore(isAgentAllowed);const screenContextAllowed = usePolicyStore(isScreenContextAllowed);+ const visionOverrideAllowed = usePolicyStore((state) =>+ isModeAllowedByPolicy(state, "llm", "providers")+ );// Display the effective value: an org that forces the feature off shows the// toggle off while the raw preference survives for when the policy lifts.const screenContextActive = voiceAgentScreenContext && screenContextAllowed;@@ -192,7 +199,7 @@ export default function DictationAgentSettings() {/></SettingsRow></SettingsPanelRow>- {screenContextActive && (+ {screenContextActive && visionOverrideAllowed && (<SettingsPanelRow><SettingsRowlabel={t("dictationAgent.screenContext.visionModel")}@@ -221,11 +228,8 @@ export default function DictationAgentSettings() {{t("dictationAgent.screenContext.relaunchHint")}</p>)}- {screenContextActive && useDictationAgentVisionModel && (- <InferenceConfigEditor- scope="dictationAgentVision"- allowedModes={["openwhispr", "providers"]}- />+ {screenContextActive && visionOverrideAllowed && useDictationAgentVisionModel && (+ <InferenceConfigEditor scope="dictationAgentVision" allowedModes={["providers"]} />)}</div>)}diff --git a/src/components/settings/InferenceConfigEditor.tsx b/src/components/settings/InferenceConfigEditor.tsxindex 6ed8a9960..a508158f7 100644--- a/src/components/settings/InferenceConfigEditor.tsx+++ b/src/components/settings/InferenceConfigEditor.tsx@@ -269,7 +269,13 @@ export default function InferenceConfigEditor({</Button></div>)}- <InferenceModeSelector modes={modes} activeMode={effectiveMode} onSelect={handleModeSelect} />+ {modes.length > 1 && (+ <InferenceModeSelector+ modes={modes}+ activeMode={effectiveMode}+ onSelect={handleModeSelect}+ />+ )}{effectiveMode === "providers" && renderModelSelector("cloud")}{effectiveMode === "local" && renderModelSelector("local")}diff --git a/src/config/inferenceScopes.ts b/src/config/inferenceScopes.tsindex 6e5633f8a..889cffbe0 100644--- a/src/config/inferenceScopes.ts+++ b/src/config/inferenceScopes.ts@@ -1,4 +1,5 @@import type { SettingsState } from "../stores/settingsStore";+import type { CloudReasonPurpose } from "../types/electron";export interface InferenceScopeStoreKeys {mode: keyof SettingsState;@@ -14,10 +15,13 @@ export interface InferenceScopeStoreKeys {export interface InferenceScopeDefinition {storeKeys: InferenceScopeStoreKeys;fallbackScope?: string;+ /** Server-side fallback chain OpenWhispr Cloud answers this scope on. */+ cloudPurpose: CloudReasonPurpose;}export const INFERENCE_SCOPES = {dictationCleanup: {+ cloudPurpose: "cleanup",storeKeys: {mode: "cleanupMode",provider: "cleanupProvider",@@ -30,6 +34,7 @@ export const INFERENCE_SCOPES = {},},dictationAgent: {+ cloudPurpose: "assistant",storeKeys: {mode: "dictationAgentMode",provider: "dictationAgentProvider",@@ -45,6 +50,7 @@ export const INFERENCE_SCOPES = {// context screenshot. Unset fields resolve to the dictationAgent scope, and// the UI offers only cloud/BYOK modes, so remoteUrl is deliberately absent.dictationAgentVision: {+ cloudPurpose: "assistant",storeKeys: {mode: "dictationAgentVisionMode",provider: "dictationAgentVisionProvider",@@ -57,6 +63,7 @@ export const INFERENCE_SCOPES = {fallbackScope: "dictationAgent",},noteFormatting: {+ cloudPurpose: "noteFormatting",storeKeys: {mode: "noteFormattingMode",provider: "noteFormattingProvider",@@ -73,6 +80,7 @@ export const INFERENCE_SCOPES = {// The voice assistant panel's spoken commands, like selection edits, run on// dictationAgent(Vision) — see resolveChatStreamingInference.chatIntelligence: {+ cloudPurpose: "assistant",storeKeys: {mode: "chatAgentMode",provider: "chatAgentProvider",@@ -85,6 +93,7 @@ export const INFERENCE_SCOPES = {},},dictationTranslation: {+ cloudPurpose: "translation",storeKeys: {mode: "translationMode",provider: "translationProvider",diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex a2abe2a4f..7b14a0d6f 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -2830,6 +2830,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const res = await window.electronAPI.cloudReason(currentText, {agentName,promptMode: "cleanup",+ purpose: "cleanup",customDictionary: getDictionaryHintWords(settings),customPrompt: this.getCustomPrompt(),language: this.getCleanupLanguage(settings),@@ -3308,6 +3309,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const res = await window.electronAPI.cloudReason(processedText, {agentName,promptMode: "cleanup",+ purpose: "cleanup",customDictionary: getDictionaryHintWords(settings),customPrompt: this.getCustomPrompt(),language: this.getCleanupLanguage(settings),@@ -4995,6 +4997,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const res = await window.electronAPI.cloudReason(finalText, {agentName,promptMode: "cleanup",+ purpose: "cleanup",customDictionary: getDictionaryHintWords(stSettings),customPrompt: this.getCustomPrompt(),language: this.getCleanupLanguage(stSettings),diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 4fd674a28..5ef6682d5 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -9091,6 +9091,7 @@ class IPCHandlers {systemPrompt: opts.systemPrompt,requestPurpose: opts.requestPurpose,promptMode: opts.promptMode,+ purpose: opts.purpose,screenContext: opts.screenContext,language: opts.language,locale: opts.locale,diff --git a/src/services/ai/inferenceProviders/openwhispr.ts b/src/services/ai/inferenceProviders/openwhispr.tsindex af1b53bf8..a4cc0d56c 100644--- a/src/services/ai/inferenceProviders/openwhispr.ts+++ b/src/services/ai/inferenceProviders/openwhispr.ts@@ -1,4 +1,5 @@import type { InferenceProvider } from "./types";+import { INFERENCE_SCOPES } from "../../../config/inferenceScopes";import { withSessionRefresh } from "../../../lib/auth";import { getSettings } from "../../../stores/settingsStore";import logger from "../../../utils/logger";@@ -20,8 +21,8 @@ export const openwhisprProvider: InferenceProvider = {// "agent" only rides with a screenshot (which already requires the new// API) — older servers reject unknown promptMode values, so plain agent// requests omit it. Explicit "cleanup" stops the server flipping to the- // action prompt on an agent-name mention. Distinct from requestPurpose,- // which declares intent for org-policy enforcement.+ // action prompt on an agent-name mention. Distinct from requestPurpose+ // (org-policy enforcement) and purpose (the server's model chain).const promptMode = config.systemPrompt? config.screenContext? "agent"@@ -36,6 +37,9 @@ export const openwhisprProvider: InferenceProvider = {systemPrompt: config.systemPrompt,requestPurpose: config.requiresAgent ? "agent" : undefined,promptMode,+ purpose: config.inferenceScope+ ? INFERENCE_SCOPES[config.inferenceScope].cloudPurpose+ : undefined,screenContext: config.screenContext,language: config.language || ctx.getPreferredLanguage(),locale: ctx.getUiLanguage(),diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.tsindex 16bb0ef1f..62e750ffe 100644--- a/src/stores/settingsStore.ts+++ b/src/stores/settingsStore.ts@@ -967,7 +967,6 @@ export interface SettingsStatesetVoiceAgentScreenContext: (value: boolean) => void;setUseDictationAgentVisionModel: (value: boolean) => void;- setDictationAgentVisionMode: (mode: InferenceMode) => void;setDictationAgentVisionProvider: (value: string) => void;setDictationAgentVisionModel: (value: string) => void;setDictationAgentVisionCloudMode: (value: string) => void;@@ -1876,11 +1875,8 @@ export const useSettingsStore = create<SettingsState>()((set, get) => ({voiceAgentScreenContext: readBoolean("voiceAgentScreenContext", false),useDictationAgentVisionModel: readBoolean("useDictationAgentVisionModel", false),- dictationAgentVisionMode: (() => {- const v = readString("dictationAgentVisionMode", "openwhispr");- if (v === "openwhispr" || v === "providers") return v as InferenceMode;- return "openwhispr" as InferenceMode;- })(),+ // Cloud already vision-routes screenshot commands, so the override is BYOK-only.+ dictationAgentVisionMode: "providers" as InferenceMode,dictationAgentVisionProvider: readString("dictationAgentVisionProvider", ""),dictationAgentVisionModel: readString("dictationAgentVisionModel", ""),dictationAgentVisionCloudMode: readString("dictationAgentVisionCloudMode", "openwhispr"),@@ -1918,9 +1914,6 @@ export const useSettingsStore = create<SettingsState>()((set, get) => ({setVoiceAgentScreenContext: createBooleanSetter("voiceAgentScreenContext"),setUseDictationAgentVisionModel: createBooleanSetter("useDictationAgentVisionModel"),- setDictationAgentVisionMode: createStringSetter("dictationAgentVisionMode") as (- mode: InferenceMode- ) => void,setDictationAgentVisionProvider: createStringSetter("dictationAgentVisionProvider"),setDictationAgentVisionModel: createStringSetter("dictationAgentVisionModel"),setDictationAgentVisionCloudMode: createStringSetter("dictationAgentVisionCloudMode"),diff --git a/src/types/electron.ts b/src/types/electron.tsindex 0ccd9bee8..9fc68307f 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -807,6 +807,8 @@ export interface ScreenRecordingAccessResult {needsRelaunch?: boolean;}+export type CloudReasonPurpose = "cleanup" | "assistant" | "translation" | "noteFormatting";+export interface ScreenContextImage {mediaType: string;/** Base64 image bytes, no data-URL prefix. */@@ -2328,6 +2330,7 @@ declare global {systemPrompt?: string;requestPurpose?: "agent";promptMode?: "cleanup" | "agent";+ purpose?: CloudReasonPurpose;screenContext?: ScreenContextImage;language?: string;locale?: string;diff --git a/test/services/openwhisprReasonPurpose.test.js b/test/services/openwhisprReasonPurpose.test.jsindex 4decb6364..561f78e85 100644--- a/test/services/openwhisprReasonPurpose.test.js+++ b/test/services/openwhisprReasonPurpose.test.js@@ -71,18 +71,32 @@ test("OpenWhispr reasoning labels only dictation-agent requests for server enfortext: "do this",model: "",agentName: "Whisper",- config: { systemPrompt: "Act on the request.", requiresAgent: true },+ config: {+ systemPrompt: "Act on the request.",+ requiresAgent: true,+ inferenceScope: "dictationAgent",+ },ctx: context,});await openwhisprProvider.call({text: "clean this",model: "",agentName: "Whisper",- config: {},+ config: { inferenceScope: "dictationCleanup" },+ ctx: context,+ });+ await openwhisprProvider.call({+ text: "bonjour",+ model: "",+ agentName: "Whisper",+ config: { systemPrompt: "Translate to English.", inferenceScope: "dictationTranslation" },ctx: context,});assert.equal(requests[0].requestPurpose, "agent");+ assert.equal(requests[0].purpose, "assistant");assert.equal(requests[1].requestPurpose, undefined);assert.equal(requests[1].promptMode, "cleanup");+ assert.equal(requests[1].purpose, "cleanup");+ assert.equal(requests[2].purpose, "translation");});