Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(agent): recognize localized wake words (#1604)

Alcahest Malus Dienheim0e681cc8030d
diff --git a/src/config/agentDetection.ts b/src/config/agentDetection.ts
index 787671ba1..35d0cf4fd 100644
--- a/src/config/agentDetection.ts
+++ b/src/config/agentDetection.ts
@@ -1,3 +1,5 @@
+import { getBaseLanguageCode } from "../utils/languageSupport";
+
function levenshteinDistance(a: string, b: string): number {
const m = a.length;
const n = b.length;
@@ -27,36 +29,113 @@ function maxEditsForLength(len: number): number {
const VOCATIVE_CUES = new Set(["hey", "hi", "hello", "ok", "okay", "yo", "please"]);
+// Localized vocatives per base dictation language, matched with the same
+// previous-token rule as the English cues. Kept short to avoid false positives.
+const LOCALIZED_VOCATIVE_CUES: Record<string, readonly string[]> = {
+ de: ["hallo", "servus"],
+ es: ["oye", "hola", "oiga"],
+ fr: ["hé", "salut"],
+ it: ["ehi", "ei", "ciao", "scusa"],
+ ja: ["ねぇ", "ねえ", "ヘイ"],
+ pt: ["ei", "olá"],
+ ru: ["привет", "эй", "слушай"],
+ zh: ["嘿", "你好", "喂"],
+};
+
+const LOCALIZED_CUE_SETS: ReadonlyMap<string, ReadonlySet<string>> = new Map(
+ Object.entries(LOCALIZED_VOCATIVE_CUES).map(([lang, cues]) => [lang, new Set(cues)])
+);
+
+const EMPTY_CUES: ReadonlySet<string> = new Set();
+
+// CJK transcripts carry no spaces ("ねぇ、Jarvis、メールを"), so fullwidth marks
+// become their ASCII equivalent plus a space, and CJK/Latin transitions split.
+const CJK_PUNCTUATION_MAP: Record<string, string> = {
+ "、": ", ",
+ "。": ". ",
+ "!": "! ",
+ "?": "? ",
+ ",": ", ",
+ ";": "; ",
+ ":": ": ",
+ "(": " (",
+ ")": ") ",
+ "「": ' "',
+ "」": '" ',
+ "『": ' "',
+ "』": '" ',
+};
+
+const CJK_PUNCTUATION_RE = new RegExp(`[${Object.keys(CJK_PUNCTUATION_MAP).join("")}]`, "g");
+const CJK_CHAR_RANGE = "\\u3040-\\u30ff\\u3400-\\u4dbf\\u4e00-\\u9fff";
+const CJK_TO_LATIN_RE = new RegExp(`([${CJK_CHAR_RANGE}])(?=[A-Za-z0-9])`, "g");
+const LATIN_TO_CJK_RE = new RegExp(`([A-Za-z0-9])(?=[${CJK_CHAR_RANGE}])`, "g");
+
+function normalizeCjkTranscript(transcript: string, agentName: string): string {
+ const escapedName = agentName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+ const agentNamePattern = new RegExp(escapedName, "giu");
+
+ return transcript
+ .normalize("NFC")
+ .replace(agentNamePattern, " $& ")
+ .replace(CJK_PUNCTUATION_RE, (ch) => CJK_PUNCTUATION_MAP[ch] ?? ch)
+ .replace(CJK_TO_LATIN_RE, "$1 ")
+ .replace(LATIN_TO_CJK_RE, "$1 ");
+}
+
+// Cues gate on a resolved language; "auto", unknown codes and junk fail closed
+// to English-only. The caller maps "auto" to its best hint (the UI language).
+function baseLanguageOf(language?: string): string | undefined {
+ if (typeof language !== "string") return undefined;
+ return getBaseLanguageCode(language.trim().toLowerCase());
+}
+
// The name only counts as addressing the agent when it starts the dictation,
// follows a greeting cue ("hey Jarvis"), or opens a new sentence. A mere
// mention elsewhere ("I showed OpenWhispr to a friend") is dictated content,
// not a command.
-function isAddressedAt(index: number, words: string[], rawWords: string[]): boolean {
+function isAddressedAt(
+ index: number,
+ words: string[],
+ rawWords: string[],
+ localizedCues: ReadonlySet<string>
+): boolean {
if (index === 0) return true;
- if (VOCATIVE_CUES.has(words[index - 1])) return true;
+ const prev = words[index - 1];
+ if (VOCATIVE_CUES.has(prev) || localizedCues.has(prev)) return true;
return /[.!?…]["')\]]*$/.test(rawWords[index - 1]);
}
-export function detectAgentName(transcript: string, agentName: string): boolean {
+export function detectAgentName(transcript: string, agentName: string, language?: string): boolean {
const name = agentName.trim();
if (!name || name.length < 2) return false;
- const nameLower = name.toLowerCase().replace(/\s+/g, "");
- const rawWords = transcript.split(/\s+/).filter(Boolean);
+ const base = baseLanguageOf(language);
+ const localizedCues = (base && LOCALIZED_CUE_SETS.get(base)) || EMPTY_CUES;
+ // Normalization is gated with the cues so non-CJK dictation stays untouched.
+ const normalizeCjk = base === "ja" || base === "zh";
+ const detectionName = normalizeCjk ? name.normalize("NFC") : name;
+ const source = normalizeCjk ? normalizeCjkTranscript(transcript, detectionName) : transcript;
+
+ const nameLower = detectionName.toLowerCase().replace(/\s+/g, "");
+ const rawWords = source.split(/\s+/).filter(Boolean);
const words = rawWords.map((w) => w.replace(/[.,!?;:'"()]/g, "").toLowerCase());
const maxEdits = maxEditsForLength(nameLower.length);
// STT may split the name across tokens ("open whispr") or mishear it, so
// compare joined windows up to the name's own token count (minimum 2)
// against the name, allowing length-scaled edits.
- const maxSpan = Math.max(2, name.split(/\s+/).length);
+ const maxSpan = Math.max(2, detectionName.split(/\s+/).length);
for (let i = 0; i < words.length; i++) {
let joined = "";
for (let span = 0; span < maxSpan && i + span < words.length; span++) {
joined += words[i + span];
if (Math.abs(joined.length - nameLower.length) > maxEdits) continue;
- if (levenshteinDistance(joined, nameLower) <= maxEdits && isAddressedAt(i, words, rawWords)) {
+ if (
+ levenshteinDistance(joined, nameLower) <= maxEdits &&
+ isAddressedAt(i, words, rawWords, localizedCues)
+ ) {
return true;
}
}
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 70cbe3db0..a7e776039 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -68,7 +68,11 @@ import {
shouldRunTranslateStep,
} from "./translationChain";
import { detectAgentName } from "../config/agentDetection";
-import { resolveDictationRouteKind, resolveAgentImageTarget } from "./dictationRouting";
+import {
+ resolveDictationRouteKind,
+ resolveAgentImageTarget,
+ resolveWakeWordLanguage,
+} from "./dictationRouting";
import {
resolveDictationAgentInference,
resolveDictationAgentVisionInference,
@@ -144,7 +148,8 @@ function resolveReasoningRoute(
agentName,
voiceAgentRequested,
translationRequested,
- screenContext
+ screenContext,
+ detectedLanguage
) {
const cleanup = selectResolvedLLMConfig(settings, "dictationCleanup");
const cleanupReachable =
@@ -160,7 +165,11 @@ function resolveReasoningRoute(
const kind = resolveDictationRouteKind({
cleanupReachable,
agentReachable: agent.reachable,
- agentInvoked: !!agentName && detectAgentName(text, agentName),
+ // A translation recording never routes to the agent, so skip the scan.
+ agentInvoked:
+ !translationRequested &&
+ !!agentName &&
+ detectAgentName(text, agentName, resolveWakeWordLanguage(settings, detectedLanguage)),
voiceAgentRequested,
translationRequested,
translationReachable: translation.reachable,
@@ -2847,7 +2856,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
agentName,
this.voiceAgentRequested,
this.translationRequested,
- screenContext
+ screenContext,
+ result.sttLanguage
);
if (this.translationRequested && route.kind !== "translation") {
this.notifyTranslationFallback("unreachable");
diff --git a/src/helpers/dictationRouting.js b/src/helpers/dictationRouting.js
index e0159b5d0..869e74b3a 100644
--- a/src/helpers/dictationRouting.js
+++ b/src/helpers/dictationRouting.js
@@ -141,6 +141,16 @@ export function resolveTranslationDisplayProvider({ translationMode, translation
return resolveModeDisplayProvider(translationMode, translationProvider);
}
+// Wake-word cues gate on the explicit dictation language, then the language
+// detected by STT, with the UI language as the final hint under auto-detect.
+export function resolveWakeWordLanguage({ preferredLanguage, uiLanguage }, detectedLanguage) {
+ const language = typeof preferredLanguage === "string" ? preferredLanguage.trim() : "";
+ if (language && language.toLowerCase() !== "auto") return language;
+ const detected = typeof detectedLanguage === "string" ? detectedLanguage.trim() : "";
+ if (detected && detected.toLowerCase() !== "auto") return detected;
+ return typeof uiLanguage === "string" ? uiLanguage : undefined;
+}
+
// Decides which reasoning path ("translation" | "agent" | "cleanup" | "skip")
// a finished dictation takes. A recording started via the voice agent hotkey
// always takes the agent path — no wake word needed — and never falls back to
diff --git a/test/helpers/agentDetection.test.js b/test/helpers/agentDetection.test.js
index b6bd5b018..54aa0e483 100644
--- a/test/helpers/agentDetection.test.js
+++ b/test/helpers/agentDetection.test.js
@@ -61,3 +61,159 @@ test("rejects empty or single-character names", async () => {
assert.equal(detectAgentName("hey there", ""), false);
assert.equal(detectAgentName("a quick note", "a"), false);
});
+
+test("matches the Italian advertised wake phrase when dictating in Italian", async () => {
+ const { detectAgentName } = await load();
+
+ // The it locale advertises "Ehi {{agentName}}".
+ assert.equal(detectAgentName("stavo pensando, Ehi Jarvis scrivi una mail", "Jarvis", "it"), true);
+ assert.equal(detectAgentName("prendi nota di questo Ciao Jarvis riassumi", "Jarvis", "it"), true);
+});
+
+test("fails closed to English-only cues when the language is auto or unset", async () => {
+ const { detectAgentName } = await load();
+
+ // The caller resolves "auto" to the UI language before calling; an
+ // unresolved language must not activate any localized cue.
+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis", "auto"), false);
+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis"), false);
+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis", "it"), true);
+});
+
+test("does not fire a foreign-language cue during English dictation", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(detectAgentName("and then oye Jarvis said something", "Jarvis", "en"), false);
+ assert.equal(detectAgentName("and then ehi Jarvis said something", "Jarvis", "en"), false);
+ assert.equal(detectAgentName("bueno pues oye Jarvis apunta esto", "Jarvis", "es"), true);
+});
+
+test("keeps English cue behavior identical for every dictation language", async () => {
+ const { detectAgentName } = await load();
+
+ for (const language of [undefined, "auto", "en", "it", "ko"]) {
+ assert.equal(
+ detectAgentName("so anyway hey Jarvis make this formal", "Jarvis", language),
+ true,
+ `language=${language}`
+ );
+ }
+});
+
+test("ignores a localized cue word mid-sentence far from the name", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(
+ detectAgentName("gli ho detto ciao mentre parlavamo del progetto di Jarvis", "Jarvis", "it"),
+ false
+ );
+ assert.equal(
+ detectAgentName("dile hola a todos antes de mencionar a Jarvis", "Jarvis", "es"),
+ false
+ );
+});
+
+test("matches the Russian advertised wake phrase including its comma", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(
+ detectAgentName("так вот Привет, Джарвис напиши письмо о бюджете", "Джарвис", "ru"),
+ true
+ );
+});
+
+test("normalizes fullwidth punctuation so CJK cues and sentence ends work", async () => {
+ const { detectAgentName } = await load();
+
+ // No spaces around the fullwidth comma, the shape CJK STT actually emits.
+ assert.equal(detectAgentName("えっと ねぇ Jarvis、メールを書いて", "Jarvis", "ja"), true);
+ assert.equal(detectAgentName("以上です。 Jarvis、続けて", "Jarvis", "ja"), true);
+ assert.equal(detectAgentName("うーん ねぇ、Jarvis、メールを書いて", "Jarvis", "ja"), true);
+});
+
+test("splits unsegmented CJK-Latin transitions so cue and name separate", async () => {
+ const { detectAgentName } = await load();
+
+ // Fully unsegmented shapes: cue glued to the name, name glued to the text.
+ assert.equal(detectAgentName("えっと ねぇJarvis、メールを書いて", "Jarvis", "ja"), true);
+ assert.equal(detectAgentName("嘿Jarvis帮我写邮件", "Jarvis", "zh"), true);
+ assert.equal(detectAgentName("「ねぇJarvisを呼んで」", "Jarvis", "ja"), true);
+});
+
+test("matches configured Unicode names in unsegmented CJK transcripts", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(detectAgentName("ねぇジャービス、メールを書いて", "ジャービス", "ja"), true);
+ assert.equal(detectAgentName("你好小助手,帮我写邮件", "小助手", "zh-CN"), true);
+ assert.equal(detectAgentName("ねぇélodieメールを書いて", "Élodie", "ja"), true);
+});
+
+test("does not treat an unsegmented CJK mention as an address", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(detectAgentName("これは小助手についての話です", "小助手", "zh-CN"), false);
+});
+
+test("leaves CJK punctuation untouched outside ja and zh dictation", async () => {
+ const { detectAgentName } = await load();
+
+ // A quoted CJK brand must not create a sentence boundary in English.
+ assert.equal(
+ detectAgentName("we compared prices to 小米。Jarvis pull up the page", "Jarvis", "en"),
+ false
+ );
+ assert.equal(
+ detectAgentName("we compared prices to 小米。Jarvis pull up the page", "Jarvis", "zh"),
+ true
+ );
+});
+
+test("leaves non-CJK Unicode normalization behavior unchanged", async () => {
+ const { detectAgentName } = await load();
+ const decomposedName = "E\u0301lodie";
+
+ assert.equal(detectAgentName(`${decomposedName} take a note`, decomposedName, "fr"), true);
+});
+
+test("maps regional language codes to their base cue set", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(detectAgentName("我想了一下 嘿 Jarvis 帮我写邮件", "Jarvis", "zh-CN"), true);
+ assert.equal(detectAgentName("我想了一下 嘿 Jarvis 幫我寫郵件", "Jarvis", "zh-TW"), true);
+});
+
+test("keeps detection English-only for languages without a localized cue set", async () => {
+ const { detectAgentName } = await load();
+
+ assert.equal(detectAgentName("well then ehi Jarvis take a note", "Jarvis", "ko"), false);
+ assert.equal(detectAgentName("well then hey Jarvis take a note", "Jarvis", "ko"), true);
+});
+
+test("every locale's advertised wake phrase triggers detection in its language", async () => {
+ const { detectAgentName } = await load();
+ const fs = require("node:fs");
+ const path = require("node:path");
+
+ const localesDir = path.join(__dirname, "..", "..", "src", "locales");
+ const locales = fs
+ .readdirSync(localesDir, { withFileTypes: true })
+ .filter((e) => e.isDirectory())
+ .map((e) => e.name);
+ assert.equal(locales.length, 10);
+
+ for (const locale of locales) {
+ const translation = JSON.parse(
+ fs.readFileSync(path.join(localesDir, locale, "translation.json"), "utf8")
+ );
+ const advertised = translation.settingsPage.agentConfig.howItWorksDescription;
+ // The advertised wake word is the last letter run before {{agentName}}.
+ const match = advertised.match(/(\p{L}+)[^\p{L}]*\{\{agentName\}\}/u);
+ assert.ok(match, `${locale}: no wake phrase before {{agentName}}`);
+ const cue = match[1];
+ assert.equal(
+ detectAgentName(`one two ${cue} Jarvis three four`, "Jarvis", locale),
+ true,
+ `${locale}: advertised cue "${cue}" does not trigger`
+ );
+ }
+});
diff --git a/test/helpers/audioManagerWakeWordLanguage.test.js b/test/helpers/audioManagerWakeWordLanguage.test.js
new file mode 100644
index 000000000..4ac47bcdc
--- /dev/null
+++ b/test/helpers/audioManagerWakeWordLanguage.test.js
@@ -0,0 +1,126 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");
+
+async function loadAudioManager(t) {
+ const { window } = installBrowserGlobals(t, {
+ initialStorage: { agentName: "Jarvis" },
+ });
+ Object.defineProperty(globalThis, "localStorage", {
+ value: window.localStorage,
+ writable: true,
+ configurable: true,
+ });
+ const originalNavigator = globalThis.navigator;
+ Object.defineProperty(globalThis, "navigator", {
+ value: { ...originalNavigator, onLine: true },
+ configurable: true,
+ });
+ t.after(() => {
+ Object.defineProperty(globalThis, "navigator", {
+ value: originalNavigator,
+ configurable: true,
+ });
+ delete globalThis.__wakeWordSettings;
+ });
+
+ const vite = await createRendererServer(t, {
+ cachePrefix: "openwhispr-wake-language-test-",
+ mockModules: {
+ "/utils/logger":
+ "export default { debug() {}, info() {}, warn() {}, error() {}, logReasoning() {} };",
+ "/stores/settingsStore": `
+ export const getSettings = () => globalThis.__wakeWordSettings;
+ export const getEffectiveCleanupModel = () => "cleanup-model";
+ export const isCloudCleanupMode = () => false;
+ export const isCloudDictationAgentMode = () => false;
+ export const isCloudTranslationMode = () => false;
+ export const selectResolvedLLMConfig = () => ({ model: "cleanup-model" });
+ `,
+ "/dictationAgentInference": `
+ export const resolveDictationAgentInference = () => ({
+ reachable: true,
+ model: "agent-model",
+ displayProvider: "test",
+ config: { provider: "test" },
+ });
+ export const resolveDictationAgentVisionInference = () => ({
+ active: false,
+ model: "",
+ config: {},
+ });
+ `,
+ "/dictationTranslationInference": `
+ export const resolveDictationTranslationInference = () => ({
+ reachable: false,
+ model: "",
+ displayProvider: "none",
+ config: {},
+ });
+ `,
+ "/config/prompts": `
+ export const resolvePrompt = () => "agent prompt";
+ export const appendScreenContextSuffix = (prompt) => prompt;
+ `,
+ "/services/ReasoningService": "export default class ReasoningService {};",
+ "/services/SyncService.js": "export const syncService = {};",
+ "/lib/auth": "export const withSessionRefresh = (fn) => fn();",
+ "/utils/permissions": "export const isAccessibilitySkipped = () => false;",
+ },
+ });
+
+ const AudioManager = (await vite.ssrLoadModule("/helpers/audioManager.js")).default;
+ return {
+ window,
+ setSettings: (settings) => {
+ globalThis.__wakeWordSettings = settings;
+ },
+ createManager: () =>
+ Object.assign(Object.create(AudioManager.prototype), {
+ skipReasoning: false,
+ voiceAgentRequested: false,
+ translationRequested: false,
+ isDictionaryEcho: () => false,
+ getWhisperPrompt: () => null,
+ assertAgentAllowedByPolicy: () => {},
+ processAgentCommand: async () => "agent output",
+ processWithReasoningModel: async () => "cleanup output",
+ finalizeChineseScript: async (text) => text,
+ }),
+ };
+}
+
+test("cloud auto-language routing uses detected speech before the UI language", async (t) => {
+ const { window, setSettings, createManager } = await loadAudioManager(t);
+ const audioBlob = {
+ type: "audio/webm",
+ size: 1024,
+ arrayBuffer: async () => new ArrayBuffer(8),
+ };
+ const settings = {
+ preferredLanguage: "auto",
+ useCleanupModel: true,
+ cleanupCloudMode: "byok",
+ cleanupDisableThinking: false,
+ customDictionary: [],
+ snippets: [],
+ };
+
+ setSettings({ ...settings, uiLanguage: "it" });
+ window.electronAPI.cloudTranscribe = async () => ({
+ success: true,
+ text: "and then ehi Jarvis said something",
+ sttLanguage: "en",
+ });
+ const englishResult = await createManager().processWithOpenWhisprCloud(audioBlob);
+ assert.equal(englishResult.text, "cleanup output");
+
+ setSettings({ ...settings, uiLanguage: "en" });
+ window.electronAPI.cloudTranscribe = async () => ({
+ success: true,
+ text: "stavo pensando ehi Jarvis scrivi una mail",
+ sttLanguage: "it",
+ });
+ const italianResult = await createManager().processWithOpenWhisprCloud(audioBlob);
+ assert.equal(italianResult.text, "agent output");
+});
diff --git a/test/helpers/dictationRouting.test.js b/test/helpers/dictationRouting.test.js
index 0ed05e1b3..58d7daa58 100644
--- a/test/helpers/dictationRouting.test.js
+++ b/test/helpers/dictationRouting.test.js
@@ -693,3 +693,39 @@ test("an override toggled on but never configured falls back to the base rules",
useVisionOverride: false,
});
});
+
+test("wake-word language follows the dictation language when it is explicit", async () => {
+ const { resolveWakeWordLanguage } = await load();
+
+ assert.equal(
+ resolveWakeWordLanguage({ preferredLanguage: "it", uiLanguage: "en" }, "fr"),
+ "it"
+ );
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "zh-CN", uiLanguage: "en" }), "zh-CN");
+});
+
+test("wake-word language uses detected speech before the UI language on auto", async () => {
+ const { resolveWakeWordLanguage } = await load();
+
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: "it" }, "en"), "en");
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "", uiLanguage: "en" }, "it"), "it");
+});
+
+test("wake-word language falls back to the UI language on auto or unset", async () => {
+ const { resolveWakeWordLanguage } = await load();
+
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: "it" }), "it");
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "", uiLanguage: "pt" }), "pt");
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: " ", uiLanguage: "de" }), "de");
+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: undefined, uiLanguage: "fr" }), "fr");
+});
+
+test("wake-word language is undefined when no usable hint exists", async () => {
+ const { resolveWakeWordLanguage } = await load();
+
+ assert.equal(
+ resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: undefined }),
+ undefined
+ );
+ assert.equal(resolveWakeWordLanguage({}), undefined);
+});