Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(assistant): BYOK-only vision override and declared cloud reasoning purpose (#2115)

Gabriel Steinbb881d10b79c
diff --git a/src/components/settings/DictationAgentSettings.tsx b/src/components/settings/DictationAgentSettings.tsx
index 601171a80..9573ecff5 100644
--- a/src/components/settings/DictationAgentSettings.tsx
+++ b/src/components/settings/DictationAgentSettings.tsx
@@ -2,7 +2,11 @@ import { useCallback, useState } from "react";
import { useTranslation } from "react-i18next";
import { Monitor } from "lucide-react";
import { useSettingsStore } from "../../stores/settingsStore";
-import { isAgentAllowed, isScreenContextAllowed } from "../../stores/policyRules";
+import {
+ isAgentAllowed,
+ isModeAllowedByPolicy,
+ isScreenContextAllowed,
+} from "../../stores/policyRules";
import { usePolicyStore } from "../../stores/policyStore";
import { useAgentName } from "../../utils/agentName";
import { useDialogs } from "../../hooks/useDialogs";
@@ -34,6 +38,9 @@ export default function DictationAgentSettings() {
} = useScreenRecordingPermission();
const agentAllowed = usePolicyStore(isAgentAllowed);
const screenContextAllowed = usePolicyStore(isScreenContextAllowed);
+ const visionOverrideAllowed = usePolicyStore((state) =>
+ isModeAllowedByPolicy(state, "llm", "providers")
+ );
// Display the effective value: an org that forces the feature off shows the
// toggle off while the raw preference survives for when the policy lifts.
const screenContextActive = voiceAgentScreenContext && screenContextAllowed;
@@ -192,7 +199,7 @@ export default function DictationAgentSettings() {
/>
</SettingsRow>
</SettingsPanelRow>
- {screenContextActive && (
+ {screenContextActive && visionOverrideAllowed && (
<SettingsPanelRow>
<SettingsRow
label={t("dictationAgent.screenContext.visionModel")}
@@ -221,11 +228,8 @@ export default function DictationAgentSettings() {
{t("dictationAgent.screenContext.relaunchHint")}
</p>
)}
- {screenContextActive && useDictationAgentVisionModel && (
- <InferenceConfigEditor
- scope="dictationAgentVision"
- allowedModes={["openwhispr", "providers"]}
- />
+ {screenContextActive && visionOverrideAllowed && useDictationAgentVisionModel && (
+ <InferenceConfigEditor scope="dictationAgentVision" allowedModes={["providers"]} />
)}
</div>
)}
diff --git a/src/components/settings/InferenceConfigEditor.tsx b/src/components/settings/InferenceConfigEditor.tsx
index 6ed8a9960..a508158f7 100644
--- a/src/components/settings/InferenceConfigEditor.tsx
+++ b/src/components/settings/InferenceConfigEditor.tsx
@@ -269,7 +269,13 @@ export default function InferenceConfigEditor({
</Button>
</div>
)}
- <InferenceModeSelector modes={modes} activeMode={effectiveMode} onSelect={handleModeSelect} />
+ {modes.length > 1 && (
+ <InferenceModeSelector
+ modes={modes}
+ activeMode={effectiveMode}
+ onSelect={handleModeSelect}
+ />
+ )}
{effectiveMode === "providers" && renderModelSelector("cloud")}
{effectiveMode === "local" && renderModelSelector("local")}
diff --git a/src/config/inferenceScopes.ts b/src/config/inferenceScopes.ts
index 6e5633f8a..889cffbe0 100644
--- a/src/config/inferenceScopes.ts
+++ b/src/config/inferenceScopes.ts
@@ -1,4 +1,5 @@
import type { SettingsState } from "../stores/settingsStore";
+import type { CloudReasonPurpose } from "../types/electron";
export interface InferenceScopeStoreKeys {
mode: keyof SettingsState;
@@ -14,10 +15,13 @@ export interface InferenceScopeStoreKeys {
export interface InferenceScopeDefinition {
storeKeys: InferenceScopeStoreKeys;
fallbackScope?: string;
+ /** Server-side fallback chain OpenWhispr Cloud answers this scope on. */
+ cloudPurpose: CloudReasonPurpose;
}
export const INFERENCE_SCOPES = {
dictationCleanup: {
+ cloudPurpose: "cleanup",
storeKeys: {
mode: "cleanupMode",
provider: "cleanupProvider",
@@ -30,6 +34,7 @@ export const INFERENCE_SCOPES = {
},
},
dictationAgent: {
+ cloudPurpose: "assistant",
storeKeys: {
mode: "dictationAgentMode",
provider: "dictationAgentProvider",
@@ -45,6 +50,7 @@ export const INFERENCE_SCOPES = {
// context screenshot. Unset fields resolve to the dictationAgent scope, and
// the UI offers only cloud/BYOK modes, so remoteUrl is deliberately absent.
dictationAgentVision: {
+ cloudPurpose: "assistant",
storeKeys: {
mode: "dictationAgentVisionMode",
provider: "dictationAgentVisionProvider",
@@ -57,6 +63,7 @@ export const INFERENCE_SCOPES = {
fallbackScope: "dictationAgent",
},
noteFormatting: {
+ cloudPurpose: "noteFormatting",
storeKeys: {
mode: "noteFormattingMode",
provider: "noteFormattingProvider",
@@ -73,6 +80,7 @@ export const INFERENCE_SCOPES = {
// The voice assistant panel's spoken commands, like selection edits, run on
// dictationAgent(Vision) — see resolveChatStreamingInference.
chatIntelligence: {
+ cloudPurpose: "assistant",
storeKeys: {
mode: "chatAgentMode",
provider: "chatAgentProvider",
@@ -85,6 +93,7 @@ export const INFERENCE_SCOPES = {
},
},
dictationTranslation: {
+ cloudPurpose: "translation",
storeKeys: {
mode: "translationMode",
provider: "translationProvider",
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index a2abe2a4f..7b14a0d6f 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -2830,6 +2830,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const res = await window.electronAPI.cloudReason(currentText, {
agentName,
promptMode: "cleanup",
+ purpose: "cleanup",
customDictionary: getDictionaryHintWords(settings),
customPrompt: this.getCustomPrompt(),
language: this.getCleanupLanguage(settings),
@@ -3308,6 +3309,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const res = await window.electronAPI.cloudReason(processedText, {
agentName,
promptMode: "cleanup",
+ purpose: "cleanup",
customDictionary: getDictionaryHintWords(settings),
customPrompt: this.getCustomPrompt(),
language: this.getCleanupLanguage(settings),
@@ -4995,6 +4997,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const res = await window.electronAPI.cloudReason(finalText, {
agentName,
promptMode: "cleanup",
+ purpose: "cleanup",
customDictionary: getDictionaryHintWords(stSettings),
customPrompt: this.getCustomPrompt(),
language: this.getCleanupLanguage(stSettings),
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 4fd674a28..5ef6682d5 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -9091,6 +9091,7 @@ class IPCHandlers {
systemPrompt: opts.systemPrompt,
requestPurpose: opts.requestPurpose,
promptMode: opts.promptMode,
+ purpose: opts.purpose,
screenContext: opts.screenContext,
language: opts.language,
locale: opts.locale,
diff --git a/src/services/ai/inferenceProviders/openwhispr.ts b/src/services/ai/inferenceProviders/openwhispr.ts
index af1b53bf8..a4cc0d56c 100644
--- a/src/services/ai/inferenceProviders/openwhispr.ts
+++ b/src/services/ai/inferenceProviders/openwhispr.ts
@@ -1,4 +1,5 @@
import type { InferenceProvider } from "./types";
+import { INFERENCE_SCOPES } from "../../../config/inferenceScopes";
import { withSessionRefresh } from "../../../lib/auth";
import { getSettings } from "../../../stores/settingsStore";
import logger from "../../../utils/logger";
@@ -20,8 +21,8 @@ export const openwhisprProvider: InferenceProvider = {
// "agent" only rides with a screenshot (which already requires the new
// API) — older servers reject unknown promptMode values, so plain agent
// requests omit it. Explicit "cleanup" stops the server flipping to the
- // action prompt on an agent-name mention. Distinct from requestPurpose,
- // which declares intent for org-policy enforcement.
+ // action prompt on an agent-name mention. Distinct from requestPurpose
+ // (org-policy enforcement) and purpose (the server's model chain).
const promptMode = config.systemPrompt
? config.screenContext
? "agent"
@@ -36,6 +37,9 @@ export const openwhisprProvider: InferenceProvider = {
systemPrompt: config.systemPrompt,
requestPurpose: config.requiresAgent ? "agent" : undefined,
promptMode,
+ purpose: config.inferenceScope
+ ? INFERENCE_SCOPES[config.inferenceScope].cloudPurpose
+ : undefined,
screenContext: config.screenContext,
language: config.language || ctx.getPreferredLanguage(),
locale: ctx.getUiLanguage(),
diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.ts
index 16bb0ef1f..62e750ffe 100644
--- a/src/stores/settingsStore.ts
+++ b/src/stores/settingsStore.ts
@@ -967,7 +967,6 @@ export interface SettingsState
setVoiceAgentScreenContext: (value: boolean) => void;
setUseDictationAgentVisionModel: (value: boolean) => void;
- setDictationAgentVisionMode: (mode: InferenceMode) => void;
setDictationAgentVisionProvider: (value: string) => void;
setDictationAgentVisionModel: (value: string) => void;
setDictationAgentVisionCloudMode: (value: string) => void;
@@ -1876,11 +1875,8 @@ export const useSettingsStore = create<SettingsState>()((set, get) => ({
voiceAgentScreenContext: readBoolean("voiceAgentScreenContext", false),
useDictationAgentVisionModel: readBoolean("useDictationAgentVisionModel", false),
- dictationAgentVisionMode: (() => {
- const v = readString("dictationAgentVisionMode", "openwhispr");
- if (v === "openwhispr" || v === "providers") return v as InferenceMode;
- return "openwhispr" as InferenceMode;
- })(),
+ // Cloud already vision-routes screenshot commands, so the override is BYOK-only.
+ dictationAgentVisionMode: "providers" as InferenceMode,
dictationAgentVisionProvider: readString("dictationAgentVisionProvider", ""),
dictationAgentVisionModel: readString("dictationAgentVisionModel", ""),
dictationAgentVisionCloudMode: readString("dictationAgentVisionCloudMode", "openwhispr"),
@@ -1918,9 +1914,6 @@ export const useSettingsStore = create<SettingsState>()((set, get) => ({
setVoiceAgentScreenContext: createBooleanSetter("voiceAgentScreenContext"),
setUseDictationAgentVisionModel: createBooleanSetter("useDictationAgentVisionModel"),
- setDictationAgentVisionMode: createStringSetter("dictationAgentVisionMode") as (
- mode: InferenceMode
- ) => void,
setDictationAgentVisionProvider: createStringSetter("dictationAgentVisionProvider"),
setDictationAgentVisionModel: createStringSetter("dictationAgentVisionModel"),
setDictationAgentVisionCloudMode: createStringSetter("dictationAgentVisionCloudMode"),
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 0ccd9bee8..9fc68307f 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -807,6 +807,8 @@ export interface ScreenRecordingAccessResult {
needsRelaunch?: boolean;
}
+export type CloudReasonPurpose = "cleanup" | "assistant" | "translation" | "noteFormatting";
+
export interface ScreenContextImage {
mediaType: string;
/** Base64 image bytes, no data-URL prefix. */
@@ -2328,6 +2330,7 @@ declare global {
systemPrompt?: string;
requestPurpose?: "agent";
promptMode?: "cleanup" | "agent";
+ purpose?: CloudReasonPurpose;
screenContext?: ScreenContextImage;
language?: string;
locale?: string;
diff --git a/test/services/openwhisprReasonPurpose.test.js b/test/services/openwhisprReasonPurpose.test.js
index 4decb6364..561f78e85 100644
--- a/test/services/openwhisprReasonPurpose.test.js
+++ b/test/services/openwhisprReasonPurpose.test.js
@@ -71,18 +71,32 @@ test("OpenWhispr reasoning labels only dictation-agent requests for server enfor
text: "do this",
model: "",
agentName: "Whisper",
- config: { systemPrompt: "Act on the request.", requiresAgent: true },
+ config: {
+ systemPrompt: "Act on the request.",
+ requiresAgent: true,
+ inferenceScope: "dictationAgent",
+ },
ctx: context,
});
await openwhisprProvider.call({
text: "clean this",
model: "",
agentName: "Whisper",
- config: {},
+ config: { inferenceScope: "dictationCleanup" },
+ ctx: context,
+ });
+ await openwhisprProvider.call({
+ text: "bonjour",
+ model: "",
+ agentName: "Whisper",
+ config: { systemPrompt: "Translate to English.", inferenceScope: "dictationTranslation" },
ctx: context,
});
assert.equal(requests[0].requestPurpose, "agent");
+ assert.equal(requests[0].purpose, "assistant");
assert.equal(requests[1].requestPurpose, undefined);
assert.equal(requests[1].promptMode, "cleanup");
+ assert.equal(requests[1].purpose, "cleanup");
+ assert.equal(requests[2].purpose, "translation");
});