Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(agent): recognize localized wake words (#1604)
Alcahest Malus Dienheim0e681cc8030d
diff --git a/src/config/agentDetection.ts b/src/config/agentDetection.tsindex 787671ba1..35d0cf4fd 100644--- a/src/config/agentDetection.ts+++ b/src/config/agentDetection.ts@@ -1,3 +1,5 @@+import { getBaseLanguageCode } from "../utils/languageSupport";+function levenshteinDistance(a: string, b: string): number {const m = a.length;const n = b.length;@@ -27,36 +29,113 @@ function maxEditsForLength(len: number): number {const VOCATIVE_CUES = new Set(["hey", "hi", "hello", "ok", "okay", "yo", "please"]);+// Localized vocatives per base dictation language, matched with the same+// previous-token rule as the English cues. Kept short to avoid false positives.+const LOCALIZED_VOCATIVE_CUES: Record<string, readonly string[]> = {+ de: ["hallo", "servus"],+ es: ["oye", "hola", "oiga"],+ fr: ["hé", "salut"],+ it: ["ehi", "ei", "ciao", "scusa"],+ ja: ["ねぇ", "ねえ", "ヘイ"],+ pt: ["ei", "olá"],+ ru: ["привет", "эй", "слушай"],+ zh: ["嘿", "你好", "喂"],+};++const LOCALIZED_CUE_SETS: ReadonlyMap<string, ReadonlySet<string>> = new Map(+ Object.entries(LOCALIZED_VOCATIVE_CUES).map(([lang, cues]) => [lang, new Set(cues)])+);++const EMPTY_CUES: ReadonlySet<string> = new Set();++// CJK transcripts carry no spaces ("ねぇ、Jarvis、メールを"), so fullwidth marks+// become their ASCII equivalent plus a space, and CJK/Latin transitions split.+const CJK_PUNCTUATION_MAP: Record<string, string> = {+ "、": ", ",+ "。": ". ",+ "!": "! ",+ "?": "? ",+ ",": ", ",+ ";": "; ",+ ":": ": ",+ "(": " (",+ ")": ") ",+ "「": ' "',+ "」": '" ',+ "『": ' "',+ "』": '" ',+};++const CJK_PUNCTUATION_RE = new RegExp(`[${Object.keys(CJK_PUNCTUATION_MAP).join("")}]`, "g");+const CJK_CHAR_RANGE = "\\u3040-\\u30ff\\u3400-\\u4dbf\\u4e00-\\u9fff";+const CJK_TO_LATIN_RE = new RegExp(`([${CJK_CHAR_RANGE}])(?=[A-Za-z0-9])`, "g");+const LATIN_TO_CJK_RE = new RegExp(`([A-Za-z0-9])(?=[${CJK_CHAR_RANGE}])`, "g");++function normalizeCjkTranscript(transcript: string, agentName: string): string {+ const escapedName = agentName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+ const agentNamePattern = new RegExp(escapedName, "giu");++ return transcript+ .normalize("NFC")+ .replace(agentNamePattern, " $& ")+ .replace(CJK_PUNCTUATION_RE, (ch) => CJK_PUNCTUATION_MAP[ch] ?? ch)+ .replace(CJK_TO_LATIN_RE, "$1 ")+ .replace(LATIN_TO_CJK_RE, "$1 ");+}++// Cues gate on a resolved language; "auto", unknown codes and junk fail closed+// to English-only. The caller maps "auto" to its best hint (the UI language).+function baseLanguageOf(language?: string): string | undefined {+ if (typeof language !== "string") return undefined;+ return getBaseLanguageCode(language.trim().toLowerCase());+}+// The name only counts as addressing the agent when it starts the dictation,// follows a greeting cue ("hey Jarvis"), or opens a new sentence. A mere// mention elsewhere ("I showed OpenWhispr to a friend") is dictated content,// not a command.-function isAddressedAt(index: number, words: string[], rawWords: string[]): boolean {+function isAddressedAt(+ index: number,+ words: string[],+ rawWords: string[],+ localizedCues: ReadonlySet<string>+): boolean {if (index === 0) return true;- if (VOCATIVE_CUES.has(words[index - 1])) return true;+ const prev = words[index - 1];+ if (VOCATIVE_CUES.has(prev) || localizedCues.has(prev)) return true;return /[.!?…]["')\]]*$/.test(rawWords[index - 1]);}-export function detectAgentName(transcript: string, agentName: string): boolean {+export function detectAgentName(transcript: string, agentName: string, language?: string): boolean {const name = agentName.trim();if (!name || name.length < 2) return false;- const nameLower = name.toLowerCase().replace(/\s+/g, "");- const rawWords = transcript.split(/\s+/).filter(Boolean);+ const base = baseLanguageOf(language);+ const localizedCues = (base && LOCALIZED_CUE_SETS.get(base)) || EMPTY_CUES;+ // Normalization is gated with the cues so non-CJK dictation stays untouched.+ const normalizeCjk = base === "ja" || base === "zh";+ const detectionName = normalizeCjk ? name.normalize("NFC") : name;+ const source = normalizeCjk ? normalizeCjkTranscript(transcript, detectionName) : transcript;++ const nameLower = detectionName.toLowerCase().replace(/\s+/g, "");+ const rawWords = source.split(/\s+/).filter(Boolean);const words = rawWords.map((w) => w.replace(/[.,!?;:'"()]/g, "").toLowerCase());const maxEdits = maxEditsForLength(nameLower.length);// STT may split the name across tokens ("open whispr") or mishear it, so// compare joined windows up to the name's own token count (minimum 2)// against the name, allowing length-scaled edits.- const maxSpan = Math.max(2, name.split(/\s+/).length);+ const maxSpan = Math.max(2, detectionName.split(/\s+/).length);for (let i = 0; i < words.length; i++) {let joined = "";for (let span = 0; span < maxSpan && i + span < words.length; span++) {joined += words[i + span];if (Math.abs(joined.length - nameLower.length) > maxEdits) continue;- if (levenshteinDistance(joined, nameLower) <= maxEdits && isAddressedAt(i, words, rawWords)) {+ if (+ levenshteinDistance(joined, nameLower) <= maxEdits &&+ isAddressedAt(i, words, rawWords, localizedCues)+ ) {return true;}}diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 70cbe3db0..a7e776039 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -68,7 +68,11 @@ import {shouldRunTranslateStep,} from "./translationChain";import { detectAgentName } from "../config/agentDetection";-import { resolveDictationRouteKind, resolveAgentImageTarget } from "./dictationRouting";+import {+ resolveDictationRouteKind,+ resolveAgentImageTarget,+ resolveWakeWordLanguage,+} from "./dictationRouting";import {resolveDictationAgentInference,resolveDictationAgentVisionInference,@@ -144,7 +148,8 @@ function resolveReasoningRoute(agentName,voiceAgentRequested,translationRequested,- screenContext+ screenContext,+ detectedLanguage) {const cleanup = selectResolvedLLMConfig(settings, "dictationCleanup");const cleanupReachable =@@ -160,7 +165,11 @@ function resolveReasoningRoute(const kind = resolveDictationRouteKind({cleanupReachable,agentReachable: agent.reachable,- agentInvoked: !!agentName && detectAgentName(text, agentName),+ // A translation recording never routes to the agent, so skip the scan.+ agentInvoked:+ !translationRequested &&+ !!agentName &&+ detectAgentName(text, agentName, resolveWakeWordLanguage(settings, detectedLanguage)),voiceAgentRequested,translationRequested,translationReachable: translation.reachable,@@ -2847,7 +2856,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);agentName,this.voiceAgentRequested,this.translationRequested,- screenContext+ screenContext,+ result.sttLanguage);if (this.translationRequested && route.kind !== "translation") {this.notifyTranslationFallback("unreachable");diff --git a/src/helpers/dictationRouting.js b/src/helpers/dictationRouting.jsindex e0159b5d0..869e74b3a 100644--- a/src/helpers/dictationRouting.js+++ b/src/helpers/dictationRouting.js@@ -141,6 +141,16 @@ export function resolveTranslationDisplayProvider({ translationMode, translationreturn resolveModeDisplayProvider(translationMode, translationProvider);}+// Wake-word cues gate on the explicit dictation language, then the language+// detected by STT, with the UI language as the final hint under auto-detect.+export function resolveWakeWordLanguage({ preferredLanguage, uiLanguage }, detectedLanguage) {+ const language = typeof preferredLanguage === "string" ? preferredLanguage.trim() : "";+ if (language && language.toLowerCase() !== "auto") return language;+ const detected = typeof detectedLanguage === "string" ? detectedLanguage.trim() : "";+ if (detected && detected.toLowerCase() !== "auto") return detected;+ return typeof uiLanguage === "string" ? uiLanguage : undefined;+}+// Decides which reasoning path ("translation" | "agent" | "cleanup" | "skip")// a finished dictation takes. A recording started via the voice agent hotkey// always takes the agent path — no wake word needed — and never falls back todiff --git a/test/helpers/agentDetection.test.js b/test/helpers/agentDetection.test.jsindex b6bd5b018..54aa0e483 100644--- a/test/helpers/agentDetection.test.js+++ b/test/helpers/agentDetection.test.js@@ -61,3 +61,159 @@ test("rejects empty or single-character names", async () => {assert.equal(detectAgentName("hey there", ""), false);assert.equal(detectAgentName("a quick note", "a"), false);});++test("matches the Italian advertised wake phrase when dictating in Italian", async () => {+ const { detectAgentName } = await load();++ // The it locale advertises "Ehi {{agentName}}".+ assert.equal(detectAgentName("stavo pensando, Ehi Jarvis scrivi una mail", "Jarvis", "it"), true);+ assert.equal(detectAgentName("prendi nota di questo Ciao Jarvis riassumi", "Jarvis", "it"), true);+});++test("fails closed to English-only cues when the language is auto or unset", async () => {+ const { detectAgentName } = await load();++ // The caller resolves "auto" to the UI language before calling; an+ // unresolved language must not activate any localized cue.+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis", "auto"), false);+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis"), false);+ assert.equal(detectAgentName("dunque vediamo ehi Jarvis prendi nota", "Jarvis", "it"), true);+});++test("does not fire a foreign-language cue during English dictation", async () => {+ const { detectAgentName } = await load();++ assert.equal(detectAgentName("and then oye Jarvis said something", "Jarvis", "en"), false);+ assert.equal(detectAgentName("and then ehi Jarvis said something", "Jarvis", "en"), false);+ assert.equal(detectAgentName("bueno pues oye Jarvis apunta esto", "Jarvis", "es"), true);+});++test("keeps English cue behavior identical for every dictation language", async () => {+ const { detectAgentName } = await load();++ for (const language of [undefined, "auto", "en", "it", "ko"]) {+ assert.equal(+ detectAgentName("so anyway hey Jarvis make this formal", "Jarvis", language),+ true,+ `language=${language}`+ );+ }+});++test("ignores a localized cue word mid-sentence far from the name", async () => {+ const { detectAgentName } = await load();++ assert.equal(+ detectAgentName("gli ho detto ciao mentre parlavamo del progetto di Jarvis", "Jarvis", "it"),+ false+ );+ assert.equal(+ detectAgentName("dile hola a todos antes de mencionar a Jarvis", "Jarvis", "es"),+ false+ );+});++test("matches the Russian advertised wake phrase including its comma", async () => {+ const { detectAgentName } = await load();++ assert.equal(+ detectAgentName("так вот Привет, Джарвис напиши письмо о бюджете", "Джарвис", "ru"),+ true+ );+});++test("normalizes fullwidth punctuation so CJK cues and sentence ends work", async () => {+ const { detectAgentName } = await load();++ // No spaces around the fullwidth comma, the shape CJK STT actually emits.+ assert.equal(detectAgentName("えっと ねぇ Jarvis、メールを書いて", "Jarvis", "ja"), true);+ assert.equal(detectAgentName("以上です。 Jarvis、続けて", "Jarvis", "ja"), true);+ assert.equal(detectAgentName("うーん ねぇ、Jarvis、メールを書いて", "Jarvis", "ja"), true);+});++test("splits unsegmented CJK-Latin transitions so cue and name separate", async () => {+ const { detectAgentName } = await load();++ // Fully unsegmented shapes: cue glued to the name, name glued to the text.+ assert.equal(detectAgentName("えっと ねぇJarvis、メールを書いて", "Jarvis", "ja"), true);+ assert.equal(detectAgentName("嘿Jarvis帮我写邮件", "Jarvis", "zh"), true);+ assert.equal(detectAgentName("「ねぇJarvisを呼んで」", "Jarvis", "ja"), true);+});++test("matches configured Unicode names in unsegmented CJK transcripts", async () => {+ const { detectAgentName } = await load();++ assert.equal(detectAgentName("ねぇジャービス、メールを書いて", "ジャービス", "ja"), true);+ assert.equal(detectAgentName("你好小助手,帮我写邮件", "小助手", "zh-CN"), true);+ assert.equal(detectAgentName("ねぇélodieメールを書いて", "Élodie", "ja"), true);+});++test("does not treat an unsegmented CJK mention as an address", async () => {+ const { detectAgentName } = await load();++ assert.equal(detectAgentName("これは小助手についての話です", "小助手", "zh-CN"), false);+});++test("leaves CJK punctuation untouched outside ja and zh dictation", async () => {+ const { detectAgentName } = await load();++ // A quoted CJK brand must not create a sentence boundary in English.+ assert.equal(+ detectAgentName("we compared prices to 小米。Jarvis pull up the page", "Jarvis", "en"),+ false+ );+ assert.equal(+ detectAgentName("we compared prices to 小米。Jarvis pull up the page", "Jarvis", "zh"),+ true+ );+});++test("leaves non-CJK Unicode normalization behavior unchanged", async () => {+ const { detectAgentName } = await load();+ const decomposedName = "E\u0301lodie";++ assert.equal(detectAgentName(`${decomposedName} take a note`, decomposedName, "fr"), true);+});++test("maps regional language codes to their base cue set", async () => {+ const { detectAgentName } = await load();++ assert.equal(detectAgentName("我想了一下 嘿 Jarvis 帮我写邮件", "Jarvis", "zh-CN"), true);+ assert.equal(detectAgentName("我想了一下 嘿 Jarvis 幫我寫郵件", "Jarvis", "zh-TW"), true);+});++test("keeps detection English-only for languages without a localized cue set", async () => {+ const { detectAgentName } = await load();++ assert.equal(detectAgentName("well then ehi Jarvis take a note", "Jarvis", "ko"), false);+ assert.equal(detectAgentName("well then hey Jarvis take a note", "Jarvis", "ko"), true);+});++test("every locale's advertised wake phrase triggers detection in its language", async () => {+ const { detectAgentName } = await load();+ const fs = require("node:fs");+ const path = require("node:path");++ const localesDir = path.join(__dirname, "..", "..", "src", "locales");+ const locales = fs+ .readdirSync(localesDir, { withFileTypes: true })+ .filter((e) => e.isDirectory())+ .map((e) => e.name);+ assert.equal(locales.length, 10);++ for (const locale of locales) {+ const translation = JSON.parse(+ fs.readFileSync(path.join(localesDir, locale, "translation.json"), "utf8")+ );+ const advertised = translation.settingsPage.agentConfig.howItWorksDescription;+ // The advertised wake word is the last letter run before {{agentName}}.+ const match = advertised.match(/(\p{L}+)[^\p{L}]*\{\{agentName\}\}/u);+ assert.ok(match, `${locale}: no wake phrase before {{agentName}}`);+ const cue = match[1];+ assert.equal(+ detectAgentName(`one two ${cue} Jarvis three four`, "Jarvis", locale),+ true,+ `${locale}: advertised cue "${cue}" does not trigger`+ );+ }+});diff --git a/test/helpers/audioManagerWakeWordLanguage.test.js b/test/helpers/audioManagerWakeWordLanguage.test.jsnew file mode 100644index 000000000..4ac47bcdc--- /dev/null+++ b/test/helpers/audioManagerWakeWordLanguage.test.js@@ -0,0 +1,126 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");++async function loadAudioManager(t) {+ const { window } = installBrowserGlobals(t, {+ initialStorage: { agentName: "Jarvis" },+ });+ Object.defineProperty(globalThis, "localStorage", {+ value: window.localStorage,+ writable: true,+ configurable: true,+ });+ const originalNavigator = globalThis.navigator;+ Object.defineProperty(globalThis, "navigator", {+ value: { ...originalNavigator, onLine: true },+ configurable: true,+ });+ t.after(() => {+ Object.defineProperty(globalThis, "navigator", {+ value: originalNavigator,+ configurable: true,+ });+ delete globalThis.__wakeWordSettings;+ });++ const vite = await createRendererServer(t, {+ cachePrefix: "openwhispr-wake-language-test-",+ mockModules: {+ "/utils/logger":+ "export default { debug() {}, info() {}, warn() {}, error() {}, logReasoning() {} };",+ "/stores/settingsStore": `+ export const getSettings = () => globalThis.__wakeWordSettings;+ export const getEffectiveCleanupModel = () => "cleanup-model";+ export const isCloudCleanupMode = () => false;+ export const isCloudDictationAgentMode = () => false;+ export const isCloudTranslationMode = () => false;+ export const selectResolvedLLMConfig = () => ({ model: "cleanup-model" });+ `,+ "/dictationAgentInference": `+ export const resolveDictationAgentInference = () => ({+ reachable: true,+ model: "agent-model",+ displayProvider: "test",+ config: { provider: "test" },+ });+ export const resolveDictationAgentVisionInference = () => ({+ active: false,+ model: "",+ config: {},+ });+ `,+ "/dictationTranslationInference": `+ export const resolveDictationTranslationInference = () => ({+ reachable: false,+ model: "",+ displayProvider: "none",+ config: {},+ });+ `,+ "/config/prompts": `+ export const resolvePrompt = () => "agent prompt";+ export const appendScreenContextSuffix = (prompt) => prompt;+ `,+ "/services/ReasoningService": "export default class ReasoningService {};",+ "/services/SyncService.js": "export const syncService = {};",+ "/lib/auth": "export const withSessionRefresh = (fn) => fn();",+ "/utils/permissions": "export const isAccessibilitySkipped = () => false;",+ },+ });++ const AudioManager = (await vite.ssrLoadModule("/helpers/audioManager.js")).default;+ return {+ window,+ setSettings: (settings) => {+ globalThis.__wakeWordSettings = settings;+ },+ createManager: () =>+ Object.assign(Object.create(AudioManager.prototype), {+ skipReasoning: false,+ voiceAgentRequested: false,+ translationRequested: false,+ isDictionaryEcho: () => false,+ getWhisperPrompt: () => null,+ assertAgentAllowedByPolicy: () => {},+ processAgentCommand: async () => "agent output",+ processWithReasoningModel: async () => "cleanup output",+ finalizeChineseScript: async (text) => text,+ }),+ };+}++test("cloud auto-language routing uses detected speech before the UI language", async (t) => {+ const { window, setSettings, createManager } = await loadAudioManager(t);+ const audioBlob = {+ type: "audio/webm",+ size: 1024,+ arrayBuffer: async () => new ArrayBuffer(8),+ };+ const settings = {+ preferredLanguage: "auto",+ useCleanupModel: true,+ cleanupCloudMode: "byok",+ cleanupDisableThinking: false,+ customDictionary: [],+ snippets: [],+ };++ setSettings({ ...settings, uiLanguage: "it" });+ window.electronAPI.cloudTranscribe = async () => ({+ success: true,+ text: "and then ehi Jarvis said something",+ sttLanguage: "en",+ });+ const englishResult = await createManager().processWithOpenWhisprCloud(audioBlob);+ assert.equal(englishResult.text, "cleanup output");++ setSettings({ ...settings, uiLanguage: "en" });+ window.electronAPI.cloudTranscribe = async () => ({+ success: true,+ text: "stavo pensando ehi Jarvis scrivi una mail",+ sttLanguage: "it",+ });+ const italianResult = await createManager().processWithOpenWhisprCloud(audioBlob);+ assert.equal(italianResult.text, "agent output");+});diff --git a/test/helpers/dictationRouting.test.js b/test/helpers/dictationRouting.test.jsindex 0ed05e1b3..58d7daa58 100644--- a/test/helpers/dictationRouting.test.js+++ b/test/helpers/dictationRouting.test.js@@ -693,3 +693,39 @@ test("an override toggled on but never configured falls back to the base rules",useVisionOverride: false,});});++test("wake-word language follows the dictation language when it is explicit", async () => {+ const { resolveWakeWordLanguage } = await load();++ assert.equal(+ resolveWakeWordLanguage({ preferredLanguage: "it", uiLanguage: "en" }, "fr"),+ "it"+ );+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "zh-CN", uiLanguage: "en" }), "zh-CN");+});++test("wake-word language uses detected speech before the UI language on auto", async () => {+ const { resolveWakeWordLanguage } = await load();++ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: "it" }, "en"), "en");+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "", uiLanguage: "en" }, "it"), "it");+});++test("wake-word language falls back to the UI language on auto or unset", async () => {+ const { resolveWakeWordLanguage } = await load();++ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: "it" }), "it");+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: "", uiLanguage: "pt" }), "pt");+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: " ", uiLanguage: "de" }), "de");+ assert.equal(resolveWakeWordLanguage({ preferredLanguage: undefined, uiLanguage: "fr" }), "fr");+});++test("wake-word language is undefined when no usable hint exists", async () => {+ const { resolveWakeWordLanguage } = await load();++ assert.equal(+ resolveWakeWordLanguage({ preferredLanguage: "auto", uiLanguage: undefined }),+ undefined+ );+ assert.equal(resolveWakeWordLanguage({}), undefined);+});