Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
import {
resolveAgentImageTarget,
resolveDictationAgentDisplayProvider,
resolveDictationAgentProvider,
resolveDictationAgentReachability,
resolveModeProvider,
resolveModeReachability,
} from "./dictationRouting.js";
import { getCloudModel, isProviderValidForMode } from "../models/ModelRegistry";
import { getManagedScopeResolution } from "../stores/enterpriseIdentityStore";
import { selectIsCloudDictationAgentMode, selectResolvedLLMConfig } from "../stores/settingsStore";
import { inheritsFallbackEndpoint } from "./reasoningRouting.js";
// The dictation agent's inference scope, shared by the dictation route in
// audioManager and the Prompt Studio test tab so a prompt test hits the same
// provider, endpoint and credentials a real dictation does.
//
// Callers must add `systemPrompt` to the config: ReasoningService treats a
// missing one as its cleanup path, which echoes the input back instead of
// running the instruction.
export function resolveDictationAgentInference(settings, { isCloudAgent = false } = {}) {
const managed = getManagedScopeResolution("dictationAgent", settings.enterpriseSetupMode);
if (managed.kind === "managed") {
return {
reachable: settings.useDictationAgent,
model: managed.model,
displayProvider: managed.provider,
config: {
inferenceScope: /** @type {const} */ ("dictationAgent"),
provider: managed.provider,
disableThinking: settings.dictationAgentDisableThinking,
reasoningEffort: settings.dictationAgentReasoningEffort,
},
};
}
const model = settings.dictationAgentModel?.trim() || "";
const isSelfHosted =
settings.dictationAgentMode === "self-hosted" && !!settings.dictationAgentRemoteUrl?.trim();
const storedProvider = settings.dictationAgentProvider?.trim() || "";
const providerForMode = isProviderValidForMode(storedProvider, settings.dictationAgentMode)
? storedProvider
: undefined;
const provider = resolveDictationAgentProvider({
isCloudAgent,
dictationAgentMode: settings.dictationAgentMode,
dictationAgentProvider: providerForMode,
});
const isCustom = settings.dictationAgentMode === "providers" && provider === "custom";
return {
reachable: resolveDictationAgentReachability({
useDictationAgent: settings.useDictationAgent,
dictationAgentMode: settings.dictationAgentMode,
dictationAgentProvider: provider,
dictationAgentModel: model,
isCloudAgent,
isSelfHostedAgent: isSelfHosted,
}),
model,
displayProvider: resolveDictationAgentDisplayProvider({
dictationAgentMode: settings.dictationAgentMode,
dictationAgentProvider: providerForMode,
}),
config: {
inferenceScope: /** @type {const} */ ("dictationAgent"),
provider,
lanUrl: isSelfHosted ? settings.dictationAgentRemoteUrl : undefined,
baseUrl: isCustom ? settings.dictationAgentCloudBaseUrl || undefined : undefined,
customApiKey:
isCustom || isSelfHosted ? settings.dictationAgentCustomApiKey || undefined : undefined,
disableThinking: settings.dictationAgentDisableThinking,
reasoningEffort: settings.dictationAgentReasoningEffort,
},
};
}
// The optional vision override: the scope a voice-agent request uses when it
// carries a screen-context screenshot. Resolved through the store selector so
// unset fields inherit the agent's own config, and treated as "active" only
// once the user has actually chosen a target — an inherited config is the
// agent scope, which the base routing rules already cover.
export function resolveDictationAgentVisionInference(settings, { isSignedIn = false } = {}) {
const resolved = selectResolvedLLMConfig(settings, "dictationAgentVision");
const mode = resolved.mode;
const isCloud = isSignedIn && mode === "openwhispr" && resolved.cloudMode === "openwhispr";
const model = resolved.model?.trim() || "";
const storedProvider = resolved.provider?.trim() || "";
const providerForMode = isProviderValidForMode(storedProvider, mode) ? storedProvider : undefined;
const provider = resolveModeProvider({ isCloud, mode, provider: providerForMode });
const isCustom = mode === "providers" && provider === "custom";
// Cloud needs no model of its own, so selecting it counts as a choice;
// otherwise the user must have picked a model for this scope specifically.
const chosen = isCloud || !!settings.dictationAgentVisionModel?.trim();
// The endpoint falls back to the agent scope, so the key that opens it must
// too — an inherited endpoint with only the vision key (or none) would call
// the agent's host with the wrong credential.
const agent = selectResolvedLLMConfig(settings, "dictationAgent");
const borrowsAgentEndpoint = inheritsFallbackEndpoint(
{ mode, cloudBaseUrl: settings.dictationAgentVisionCloudBaseUrl },
agent.mode
);
const customApiKey =
resolved.customApiKey || (borrowsAgentEndpoint ? agent.customApiKey || "" : "");
return {
active:
!!settings.useDictationAgentVisionModel &&
chosen &&
resolveModeReachability({ mode, provider, model, isCloud, isSelfHosted: false }),
mode,
// Cloud picks the model server-side from its vision chain.
model: isCloud ? "" : model,
config: {
// The vision override is the agent's image lane: policy and managed
// enforcement must judge it as the agent scope, not dictation cleanup.
inferenceScope: /** @type {const} */ ("dictationAgent"),
provider,
baseUrl: isCustom ? resolved.cloudBaseUrl || undefined : undefined,
customApiKey: isCustom ? customApiKey || undefined : undefined,
disableThinking: resolved.disableThinking,
reasoningEffort: resolved.reasoningEffort,
},
};
}
/**
* What a chat conversation streams on, and whether its screenshot rides along.
*
* Typed chat surfaces resolve the Chat scope. The voice assistant panel resolves
* the Voice Assistant scope — the tab that also governs selection edits — so the
* model picked there answers spoken commands; while that scope is unreachable
* (assistant toggled off, a BYOK mode with no model) the panel falls back to
* Chat, its previous home. Profiles from before onboarding fanned selections out
* are seeded from Chat on upgrade (settingsStore), so the fallback is not what
* keeps them off the cloud default.
*
* Screenshots follow the dictation route's rules (resolveAgentImageTarget): a
* configured vision override is trusted to see images and swapped in; an
* override that cannot drops the screenshot rather than redirecting it to a
* model the user did not choose; the base scope attaches only where its
* provider is image-wired and the registry says its model has vision, or on
* OpenWhispr Cloud, which vision-routes server-side.
*
* `isProviderImageWired` is injected: the provider registry reads Vite env at
* load, which this helper's callers and tests do not all have.
*
* @param {import("../stores/settingsStore").SettingsState} settings
* @param {{
* inferenceScope?: "chatIntelligence" | "dictationAgent",
* hasScreenContext?: boolean,
* isProviderImageWired?: (providerId: string | undefined) => boolean,
* }} [options]
* @returns {{
* config: import("../stores/settingsStore").ResolvedLLMConfig,
* attachScreenContext: boolean,
* }}
*/
export function resolveChatStreamingInference(
settings,
{
inferenceScope = "chatIntelligence",
hasScreenContext = false,
isProviderImageWired = () => false,
} = {}
) {
const onAgentScope =
inferenceScope === "dictationAgent" &&
resolveDictationAgentInference(settings, {
isCloudAgent: selectIsCloudDictationAgentMode(settings),
}).reachable;
const config = selectResolvedLLMConfig(
settings,
onAgentScope ? "dictationAgent" : "chatIntelligence"
);
const isCloud = !!settings.isSignedIn && config.mode === "openwhispr";
const vision =
onAgentScope && hasScreenContext
? resolveDictationAgentVisionInference(settings, { isSignedIn: !!settings.isSignedIn })
: null;
const { attach, useVisionOverride } = resolveAgentImageTarget({
hasScreenContext,
visionOverrideActive: !!vision?.active,
visionProviderImageWired: isProviderImageWired(vision?.config.provider),
baseProviderImageWired: isProviderImageWired(
resolveModeProvider({ isCloud, mode: config.mode, provider: config.provider })
),
isCloudAgent: isCloud,
baseModelSupportsVision: !!getCloudModel(config.model, config.provider)?.supportsVision,
});
if (!useVisionOverride) return { config, attachScreenContext: attach };
return {
config: {
scope: "dictationAgentVision",
mode: vision.mode,
provider: vision.config.provider,
model: vision.model,
cloudBaseUrl: vision.config.baseUrl,
customApiKey: vision.config.customApiKey,
disableThinking: vision.config.disableThinking,
reasoningEffort: vision.config.reasoningEffort,
},
attachScreenContext: true,
};
}