Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(prompt-studio): agent prompt test runs against the cleanup scope and echoes its input (#1359)

William Gerstung143971d3aa97
diff --git a/src/components/ui/PromptStudio.tsx b/src/components/ui/PromptStudio.tsx
index b3bdeab84..acab347bd 100644
--- a/src/components/ui/PromptStudio.tsx
+++ b/src/components/ui/PromptStudio.tsx
@@ -23,10 +23,12 @@ import { getDefaultPromptText, resolvePrompt, type PromptKind } from "../../conf
import {
useSettingsStore,
selectIsCloudCleanupMode,
+ selectIsCloudDictationAgentMode,
selectIsCloudTranslationMode,
} from "../../stores/settingsStore";
import { getLanguageLabel } from "../../utils/languageSupport";
import { getDictionaryHintWords } from "../../utils/snippets";
+import { resolveDictationAgentInference } from "../../helpers/dictationAgentInference";
interface PromptStudioProps {
className?: string;
@@ -77,6 +79,11 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
const useCleanupModel = useSettingsStore((s) => s.useCleanupModel);
const cleanupModel = useSettingsStore((s) => s.cleanupModel);
+ const isCloudDictationAgent = useSettingsStore(selectIsCloudDictationAgentMode);
+ const useDictationAgent = useSettingsStore((s) => s.useDictationAgent);
+ const dictationAgentProvider = useSettingsStore((s) => s.dictationAgentProvider);
+ const dictationAgentModel = useSettingsStore((s) => s.dictationAgentModel);
+
const isCloudTranslation = useSettingsStore(selectIsCloudTranslationMode);
const useDictationTranslation = useSettingsStore((s) => s.useDictationTranslation);
const translationMode = useSettingsStore((s) => s.translationMode);
@@ -89,6 +96,7 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
const translationTargetLanguage = useSettingsStore((s) => s.translationTargetLanguage);
const isTranslate = kind === "translate";
+ const isAgent = kind === "dictationAgent";
const customPrompt = useSettingsStore((s) => s.customPrompts[kind]);
const setCustomPrompt = useSettingsStore((s) => s.setCustomPrompt);
@@ -173,6 +181,43 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
return;
}
+ // The agent runs on its own inference scope; falling through to the cleanup
+ // branch would test the cleanup provider with the cleanup prompt.
+ if (isAgent) {
+ if (!useDictationAgent) {
+ setTestResult(t("promptStudio.test.agentDisabled"));
+ return;
+ }
+
+ const settings = useSettingsStore.getState();
+ const agent = resolveDictationAgentInference(settings, {
+ isCloudAgent: isCloudDictationAgent,
+ });
+
+ if (!agent.reachable) {
+ setTestResult(t("promptStudio.test.noModelSelected"));
+ return;
+ }
+
+ const previous = customPrompt;
+ setCustomPrompt(kind, editedPrompt);
+ try {
+ const result = await ReasoningService.processText(testText, agent.model, agentName, {
+ ...agent.config,
+ systemPrompt: resolvePrompt("dictationAgent", {
+ agentName,
+ language: settings.preferredLanguage,
+ customDictionary: getDictionaryHintWords(settings),
+ uiLanguage,
+ }),
+ });
+ setTestResult(result);
+ } finally {
+ setCustomPrompt(kind, previous);
+ }
+ return;
+ }
+
const cleanupProvider = isCloudMode
? "openwhispr"
: cleanupModel
@@ -384,15 +429,25 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
{/* ── Test Tab ── */}
{activeTab === "test" &&
(() => {
- const testIsCloud = isTranslate ? isCloudTranslation : isCloudMode;
- const testModel = isTranslate ? translationModel : cleanupModel;
+ // Each kind reports the scope that actually runs it.
+ const testIsCloud = isTranslate
+ ? isCloudTranslation
+ : isAgent
+ ? isCloudDictationAgent
+ : isCloudMode;
+ const testModel = isTranslate
+ ? translationModel
+ : isAgent
+ ? dictationAgentModel
+ : cleanupModel;
+ const scopeProvider = isTranslate
+ ? translationProvider.trim()
+ : isAgent
+ ? dictationAgentProvider.trim()
+ : "";
const testProvider = testIsCloud
? "openwhispr"
- : isTranslate && translationProvider.trim()
- ? translationProvider.trim()
- : testModel
- ? getModelProvider(testModel)
- : "openai";
+ : scopeProvider || (testModel ? getModelProvider(testModel) : "openai");
const providerConfig = PROVIDER_CONFIG[testProvider] || {
label: testProvider.charAt(0).toUpperCase() + testProvider.slice(1),
};
@@ -407,7 +462,7 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
return (
<div className="divide-y divide-border/40 dark:divide-border-subtle">
- {!isTranslate && !useCleanupModel && (
+ {!isTranslate && !isAgent && !useCleanupModel && (
<div className="px-5 py-4">
<div className="rounded-lg border border-warning/20 bg-warning/5 dark:bg-warning/10 px-4 py-3">
<div className="flex items-start gap-2.5">
@@ -452,14 +507,14 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
{testText && (
<span
className={`text-xs font-medium uppercase tracking-wider px-1.5 py-px rounded ${
- isTranslate || isAgentAddressed
+ isTranslate || isAgent || isAgentAddressed
? "bg-primary/10 text-primary dark:bg-primary/15"
: "bg-muted text-muted-foreground"
}`}
>
{isTranslate
? t("promptStudio.test.translation")
- : isAgentAddressed
+ : isAgent || isAgentAddressed
? t("promptStudio.test.instruction")
: t("promptStudio.test.cleanup")}
</span>
@@ -472,19 +527,26 @@ export default function PromptStudio({ className = "", kind = "cleanup" }: Promp
className="text-xs"
placeholder={t("promptStudio.test.inputPlaceholder")}
/>
- <p className="text-xs text-muted-foreground/40 mt-1.5">
- {isTranslate
- ? t("promptStudio.test.translateHint", {
- language: getLanguageLabel(translationTargetLanguage),
- })
- : t("promptStudio.test.addressHint", { agentName })}
- </p>
+ {/* The agent tab always runs the agent prompt, addressed or not. */}
+ {!isAgent && (
+ <p className="text-xs text-muted-foreground/40 mt-1.5">
+ {isTranslate
+ ? t("promptStudio.test.translateHint", {
+ language: getLanguageLabel(translationTargetLanguage),
+ })
+ : t("promptStudio.test.addressHint", { agentName })}
+ </p>
+ )}
</div>
<div className="px-5 py-4">
<Button
onClick={testPrompt}
- disabled={!testText.trim() || isLoading || (!isTranslate && !useCleanupModel)}
+ disabled={
+ !testText.trim() ||
+ isLoading ||
+ (!isTranslate && !isAgent && !useCleanupModel)
+ }
size="sm"
className="w-full"
>
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 36e5d14d9..0515a9e8e 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -46,9 +46,9 @@ import {
import { detectAgentName } from "../config/agentDetection";
import {
resolveDictationRouteKind,
- resolveDictationAgentReachability,
resolveDictationTranslationReachability,
} from "./dictationRouting";
+import { resolveDictationAgentInference } from "./dictationAgentInference";
import { resolvePrompt } from "../config/prompts";
import { syncService } from "../services/SyncService.js";
import { evaluateFinishedRecording } from "./recordingValidation";
@@ -63,13 +63,8 @@ const PREVIEW_FLUSH_WATCHDOG_MS = 1000;
const REALTIME_MODELS = new Set(["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]);
function dictationAgentReachable(settings) {
- return resolveDictationAgentReachability({
- useDictationAgent: settings.useDictationAgent,
- dictationAgentModel: settings.dictationAgentModel,
- isCloudAgent: isCloudDictationAgentMode(),
- isSelfHostedAgent:
- settings.dictationAgentMode === "self-hosted" && !!settings.dictationAgentRemoteUrl?.trim(),
- });
+ return resolveDictationAgentInference(settings, { isCloudAgent: isCloudDictationAgentMode() })
+ .reachable;
}
function translationChainReachable(settings) {
@@ -93,15 +88,8 @@ function resolveReasoningRoute(
) {
const cleanupReachable =
!!settings.useCleanupModel && (!!settings.cleanupModel?.trim() || isCloudCleanupMode());
- const agentModel = settings.dictationAgentModel?.trim() || "";
- const isCloudAgent = isCloudDictationAgentMode();
- const isSelfHostedAgent =
- settings.dictationAgentMode === "self-hosted" && !!settings.dictationAgentRemoteUrl?.trim();
- const agentReachable = resolveDictationAgentReachability({
- useDictationAgent: settings.useDictationAgent,
- dictationAgentModel: agentModel,
- isCloudAgent,
- isSelfHostedAgent,
+ const agent = resolveDictationAgentInference(settings, {
+ isCloudAgent: isCloudDictationAgentMode(),
});
const isCloudTranslation = isCloudTranslationMode();
@@ -117,7 +105,7 @@ function resolveReasoningRoute(
const kind = resolveDictationRouteKind({
cleanupReachable,
- agentReachable,
+ agentReachable: agent.reachable,
agentInvoked: !!agentName && detectAgentName(text, agentName),
voiceAgentRequested,
translationRequested,
@@ -164,22 +152,11 @@ function resolveReasoningRoute(
};
}
if (kind === "agent") {
- const provider = isCloudAgent
- ? "openwhispr"
- : settings.dictationAgentProvider?.trim() || undefined;
- const isCustomAgent = settings.dictationAgentMode === "providers" && provider === "custom";
return {
kind: "agent",
- model: agentModel,
+ model: agent.model,
config: {
- provider,
- lanUrl: isSelfHostedAgent ? settings.dictationAgentRemoteUrl : undefined,
- baseUrl: isCustomAgent ? settings.dictationAgentCloudBaseUrl || undefined : undefined,
- customApiKey:
- isCustomAgent || isSelfHostedAgent
- ? settings.dictationAgentCustomApiKey || undefined
- : undefined,
- disableThinking: settings.dictationAgentDisableThinking,
+ ...agent.config,
systemPrompt: resolvePrompt("dictationAgent", {
agentName,
language: settings.preferredLanguage,
diff --git a/src/helpers/dictationAgentInference.js b/src/helpers/dictationAgentInference.js
new file mode 100644
index 000000000..5468277c8
--- /dev/null
+++ b/src/helpers/dictationAgentInference.js
@@ -0,0 +1,36 @@
+import { resolveDictationAgentReachability } from "./dictationRouting.js";
+
+// The dictation agent's inference scope, shared by the dictation route in
+// audioManager and the Prompt Studio test tab so a prompt test hits the same
+// provider, endpoint and credentials a real dictation does.
+//
+// Callers must add `systemPrompt` to the config: ReasoningService treats a
+// missing one as its cleanup path, which echoes the input back instead of
+// running the instruction.
+export function resolveDictationAgentInference(settings, { isCloudAgent = false } = {}) {
+ const model = settings.dictationAgentModel?.trim() || "";
+ const isSelfHosted =
+ settings.dictationAgentMode === "self-hosted" && !!settings.dictationAgentRemoteUrl?.trim();
+ const provider = isCloudAgent
+ ? "openwhispr"
+ : settings.dictationAgentProvider?.trim() || undefined;
+ const isCustom = settings.dictationAgentMode === "providers" && provider === "custom";
+
+ return {
+ reachable: resolveDictationAgentReachability({
+ useDictationAgent: settings.useDictationAgent,
+ dictationAgentModel: model,
+ isCloudAgent,
+ isSelfHostedAgent: isSelfHosted,
+ }),
+ model,
+ config: {
+ provider,
+ lanUrl: isSelfHosted ? settings.dictationAgentRemoteUrl : undefined,
+ baseUrl: isCustom ? settings.dictationAgentCloudBaseUrl || undefined : undefined,
+ customApiKey:
+ isCustom || isSelfHosted ? settings.dictationAgentCustomApiKey || undefined : undefined,
+ disableThinking: settings.dictationAgentDisableThinking,
+ },
+ };
+}
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index ab7b15fc3..2c96fc675 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "Benutzerdefinierter Endpoint",
"apiKeyMissing": "{{provider}} API-Schlüssel nicht konfiguriert. Fügen Sie ihn in den KI-Modelle-Einstellungen hinzu.",
"translationDisabled": "Aktiviere die Diktatübersetzung, um diesen Prompt zu testen",
- "noTargetLanguage": "Wähle zuerst eine Zielsprache"
+ "noTargetLanguage": "Wähle zuerst eine Zielsprache",
+ "agentDisabled": "Aktiviere den Sprachagenten, um diesen Prompt zu testen"
},
"view": {
"customPrompt": "Benutzerdefinierter Prompt",
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 886fcb56e..c2694df21 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -1143,7 +1143,8 @@
"customEndpoint": "Custom endpoint",
"apiKeyMissing": "{{provider}} API key not configured. Add it in AI Models settings.",
"translationDisabled": "Enable Dictation Translation to test this prompt",
- "noTargetLanguage": "Pick a target language first"
+ "noTargetLanguage": "Pick a target language first",
+ "agentDisabled": "Enable the voice agent to test this prompt"
},
"view": {
"customPrompt": "Custom prompt",
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 3776e406e..680fd395f 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -1103,7 +1103,8 @@
"customEndpoint": "Endpoint personalizado",
"apiKeyMissing": "La clave API de {{provider}} no está configurada. Agrégala en la configuración de Modelos de IA.",
"translationDisabled": "Activa la traducción de dictado para probar este prompt",
- "noTargetLanguage": "Elige primero un idioma de destino"
+ "noTargetLanguage": "Elige primero un idioma de destino",
+ "agentDisabled": "Activa el agente de voz para probar este prompt"
},
"view": {
"customPrompt": "Prompt personalizado",
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index bad8bced3..1895c0d4d 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -1103,7 +1103,8 @@
"customEndpoint": "Endpoint personnalisé",
"apiKeyMissing": "Clé API {{provider}} non configurée. Ajoutez-la dans les paramètres Modèles IA.",
"translationDisabled": "Activez la traduction de dictée pour tester ce prompt",
- "noTargetLanguage": "Choisissez d’abord une langue cible"
+ "noTargetLanguage": "Choisissez d’abord une langue cible",
+ "agentDisabled": "Activez l'agent vocal pour tester ce prompt"
},
"view": {
"customPrompt": "Prompt personnalisé",
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index ffd1f4c28..3401cb500 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "Endpoint personalizzato",
"apiKeyMissing": "Chiave API di {{provider}} non configurata. Aggiungila nelle impostazioni Modelli IA.",
"translationDisabled": "Attiva la traduzione della dettatura per testare questo prompt",
- "noTargetLanguage": "Scegli prima una lingua di destinazione"
+ "noTargetLanguage": "Scegli prima una lingua di destinazione",
+ "agentDisabled": "Abilita l'agente vocale per testare questo prompt"
},
"view": {
"customPrompt": "Prompt personalizzato",
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index 94f3072c6..c8e3d3d44 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "カスタムエンドポイント",
"apiKeyMissing": "{{provider}} の API キーが未設定です。AI モデル設定で追加してください。",
"translationDisabled": "このプロンプトをテストするには音声入力翻訳を有効にしてください",
- "noTargetLanguage": "先に翻訳先の言語を選択してください"
+ "noTargetLanguage": "先に翻訳先の言語を選択してください",
+ "agentDisabled": "このプロンプトをテストするには音声エージェントを有効にしてください"
},
"view": {
"customPrompt": "カスタムプロンプト",
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index f7fc3521c..86f0551c6 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -1034,7 +1034,8 @@
"customEndpoint": "Endpoint personalizado",
"apiKeyMissing": "Chave API do {{provider}} não configurada. Adicione nas configurações de Modelos de IA.",
"translationDisabled": "Ative a tradução de ditado para testar este prompt",
- "noTargetLanguage": "Escolha primeiro um idioma de destino"
+ "noTargetLanguage": "Escolha primeiro um idioma de destino",
+ "agentDisabled": "Ative o agente de voz para testar este prompt"
},
"view": {
"customPrompt": "Prompt personalizado",
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index 9437c541e..a1b30320a 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "Пользовательская конечная точка",
"apiKeyMissing": "API-ключ {{provider}} не настроен. Добавьте его в настройках ИИ-моделей.",
"translationDisabled": "Включите перевод диктовки, чтобы протестировать этот промпт",
- "noTargetLanguage": "Сначала выберите язык перевода"
+ "noTargetLanguage": "Сначала выберите язык перевода",
+ "agentDisabled": "Включите голосового агента, чтобы протестировать этот промпт"
},
"view": {
"customPrompt": "Пользовательский промпт",
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index f51f24ec4..2c112a0bf 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "自定义端点",
"apiKeyMissing": "未配置 {{provider}} API 密钥,请在 AI 模型设置中添加。",
"translationDisabled": "启用听写翻译以测试此提示词",
- "noTargetLanguage": "请先选择目标语言"
+ "noTargetLanguage": "请先选择目标语言",
+ "agentDisabled": "启用语音助手以测试此提示词"
},
"view": {
"customPrompt": "自定义提示词",
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index d084fa78b..ec0344bad 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -1055,7 +1055,8 @@
"customEndpoint": "自訂端點",
"apiKeyMissing": "未設定 {{provider}} API 金鑰。請在 AI 模型設定中新增。",
"translationDisabled": "啟用聽寫翻譯以測試此提示詞",
- "noTargetLanguage": "請先選擇目標語言"
+ "noTargetLanguage": "請先選擇目標語言",
+ "agentDisabled": "啟用語音助理以測試此提示詞"
},
"view": {
"customPrompt": "自訂提示詞",
diff --git a/test/helpers/dictationAgentInference.test.js b/test/helpers/dictationAgentInference.test.js
new file mode 100644
index 000000000..06e55f4d1
--- /dev/null
+++ b/test/helpers/dictationAgentInference.test.js
@@ -0,0 +1,101 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const load = () => import("../../src/helpers/dictationAgentInference.js");
+
+const baseSettings = {
+ useDictationAgent: true,
+ dictationAgentMode: "providers",
+ dictationAgentProvider: "llama",
+ dictationAgentModel: "llama-3.1-8b-instruct-q4_k_m",
+ dictationAgentRemoteUrl: "",
+ dictationAgentCloudBaseUrl: "",
+ dictationAgentCustomApiKey: "",
+ dictationAgentDisableThinking: true,
+};
+
+test("uses the dictation agent scope, not the cleanup scope", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({
+ ...baseSettings,
+ // Cleanup is configured completely differently; it must not leak through.
+ cleanupProvider: "openai",
+ cleanupModel: "gpt-4.1-mini",
+ });
+
+ assert.equal(result.model, "llama-3.1-8b-instruct-q4_k_m");
+ assert.equal(result.config.provider, "llama");
+});
+
+test("a local agent is unreachable without an explicit model", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({ ...baseSettings, dictationAgentModel: "" });
+
+ assert.equal(result.reachable, false);
+});
+
+test("self-hosted is reachable with no model and forwards the LAN url", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({
+ ...baseSettings,
+ dictationAgentMode: "self-hosted",
+ dictationAgentModel: "",
+ dictationAgentRemoteUrl: "http://127.0.0.1:8080/v1",
+ dictationAgentCustomApiKey: "test-key",
+ });
+
+ assert.equal(result.reachable, true);
+ assert.equal(result.config.lanUrl, "http://127.0.0.1:8080/v1");
+ assert.equal(result.config.customApiKey, "test-key");
+});
+
+test("cloud is reachable with no model", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference(
+ { ...baseSettings, dictationAgentModel: "" },
+ { isCloudAgent: true }
+ );
+
+ assert.equal(result.reachable, true);
+ assert.equal(result.model, "");
+ assert.equal(result.config.provider, "openwhispr");
+});
+
+test("a custom provider forwards its base url and api key", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({
+ ...baseSettings,
+ dictationAgentProvider: "custom",
+ dictationAgentCloudBaseUrl: "https://example.test/v1",
+ dictationAgentCustomApiKey: "sk-test",
+ });
+
+ assert.equal(result.config.baseUrl, "https://example.test/v1");
+ assert.equal(result.config.customApiKey, "sk-test");
+});
+
+test("a non-custom provider leaks neither base url nor api key", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({
+ ...baseSettings,
+ dictationAgentCloudBaseUrl: "https://example.test/v1",
+ dictationAgentCustomApiKey: "sk-test",
+ });
+
+ assert.equal(result.config.baseUrl, undefined);
+ assert.equal(result.config.customApiKey, undefined);
+});
+
+test("a disabled agent is unreachable", async () => {
+ const { resolveDictationAgentInference } = await load();
+
+ const result = resolveDictationAgentInference({ ...baseSettings, useDictationAgent: false });
+
+ assert.equal(result.reachable, false);
+});