A free, open source, and extensible speech-to-text application that works completely offline.
feat(transcription): add word correction with phonetic matching
Vlad Gerasimov003bf23402d7
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lockindex 9203b8fe..6ccf0619 100644--- a/src-tauri/Cargo.lock+++ b/src-tauri/Cargo.lock@@ -1105,7 +1105,7 @@ dependencies = ["libc","option-ext","redox_users 0.5.0",- "windows-sys 0.59.0",+ "windows-sys 0.60.2",][[package]]@@ -2001,6 +2001,7 @@ dependencies = ["futures-util","hound","log",+ "natural","once_cell","ort-sys","rdev",@@ -2010,6 +2011,7 @@ dependencies = ["rustfft","serde","serde_json",+ "strsim","tauri","tauri-build","tauri-plugin-autostart",@@ -2962,6 +2964,15 @@ dependencies = ["tempfile",]+[[package]]+name = "natural"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65e6b44f8ddc659cde3555e0140d3441ad26cb03a1410774af1f9a19097c1867"+dependencies = [+ "rust-stemmers",+]+[[package]]name = "ndarray"version = "0.16.1"@@ -4365,6 +4376,16 @@ dependencies = ["realfft",]+[[package]]+name = "rust-stemmers"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"+dependencies = [+ "serde",+ "serde_derive",+]+[[package]]name = "rustc-demangle"version = "0.1.24"diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.tomlindex 9b31d845..89f25ac1 100644--- a/src-tauri/Cargo.toml+++ b/src-tauri/Cargo.toml@@ -57,6 +57,8 @@ reqwest = { version = "0.11", features = ["json", "stream"] }futures-util = "0.3"tauri-plugin-fs = "2"rustfft = "6.4.0"+strsim = "0.11.0"+natural = "0.5.0"[dependencies.ort-sys]version = "=2.0.0-rc.9"diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex e220fac4..9bc07029 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -187,6 +187,7 @@ pub fn run() {shortcut::change_selected_language_setting,shortcut::change_overlay_position_setting,shortcut::change_debug_mode_setting,+ shortcut::update_correct_words,shortcut::suspend_binding,shortcut::resume_binding,trigger_update_check,diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rsindex bfd3e797..13fc4c20 100644--- a/src-tauri/src/managers/transcription.rs+++ b/src-tauri/src/managers/transcription.rs@@ -1,8 +1,10 @@use crate::managers::model::ModelManager;use crate::settings::get_settings;use anyhow::Result;+use natural::phonetics::soundex;use serde::Serialize;use std::sync::{Arc, Mutex};+use strsim::levenshtein;use tauri::{App, AppHandle, Emitter, Manager};use whisper_rs::install_whisper_log_trampoline;use whisper_rs::{@@ -25,6 +27,82 @@ pub struct TranscriptionManager {current_model_id: Mutex<Option<String>>,}+fn correct_words(text: &str, correct_words: &[String]) -> String {+ if correct_words.is_empty() {+ return text.to_string();+ }++ let words: Vec<&str> = text.split_whitespace().collect();+ let mut corrected_words = Vec::new();++ for word in words {+ let cleaned_word = word.trim_matches(|c: char| !c.is_alphabetic()).to_lowercase();++ if cleaned_word.is_empty() {+ corrected_words.push(word.to_string());+ continue;+ }++ let mut best_match: Option<&String> = None;+ let mut best_score = f64::MAX;++ for correct_word in correct_words {+ let correct_word_lower = correct_word.to_lowercase();++ // Calculate Levenshtein distance (normalized by length)+ let levenshtein_dist = levenshtein(&cleaned_word, &correct_word_lower);+ let max_len = cleaned_word.len().max(correct_word_lower.len()) as f64;+ let levenshtein_score = if max_len > 0.0 { levenshtein_dist as f64 / max_len } else { 1.0 };++ // Calculate phonetic similarity using Soundex+ let phonetic_match = soundex(&cleaned_word, &correct_word_lower);++ // Combine scores: favor phonetic matches, but also consider string similarity+ let combined_score = if phonetic_match {+ levenshtein_score * 0.3 // Give significant boost to phonetic matches+ } else {+ levenshtein_score+ };++ // Accept if the score is good enough (threshold: 0.4 for good matches)+ if combined_score < 0.4 && combined_score < best_score {+ best_match = Some(correct_word);+ best_score = combined_score;+ }+ }++ if let Some(replacement) = best_match {+ // Preserve the original case pattern as much as possible+ let corrected = if word.chars().all(|c| c.is_uppercase()) {+ replacement.to_uppercase()+ } else if word.chars().next().map_or(false, |c| c.is_uppercase()) {+ let mut chars: Vec<char> = replacement.chars().collect();+ if let Some(first_char) = chars.get_mut(0) {+ *first_char = first_char.to_uppercase().next().unwrap_or(*first_char);+ }+ chars.into_iter().collect()+ } else {+ replacement.clone()+ };++ // Preserve punctuation from original word+ let original_suffix: String = word.chars().rev()+ .take_while(|c| !c.is_alphabetic())+ .collect::<String>()+ .chars().rev().collect();+ let original_prefix: String = word.chars()+ .take_while(|c| !c.is_alphabetic())+ .collect();++ corrected_words.push(format!("{}{}{}", original_prefix, corrected, original_suffix));+ } else {+ corrected_words.push(word.to_string());+ }+ }++ corrected_words.join(" ")+}+impl TranscriptionManager {pub fn new(app: &App, model_manager: Arc<ModelManager>) -> Result<Self> {let app_handle = app.app_handle().clone();@@ -207,6 +285,13 @@ impl TranscriptionManager {result.push_str(&segment);}+ // Apply word correction if correct words are configured+ let corrected_result = if !settings.correct_words.is_empty() {+ correct_words(&result, &settings.correct_words)+ } else {+ result+ };+let et = std::time::Instant::now();let translation_note = if settings.translate_to_english {" (translated)"@@ -215,6 +300,6 @@ impl TranscriptionManager {};println!("\ntook {}ms{}", (et - st).as_millis(), translation_note);- Ok(result.trim().to_string())+ Ok(corrected_result.trim().to_string())}}diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rsindex 99d1520a..39863d3d 100644--- a/src-tauri/src/settings.rs+++ b/src-tauri/src/settings.rs@@ -42,6 +42,8 @@ pub struct AppSettings {pub overlay_position: OverlayPosition,#[serde(default = "default_debug_mode")]pub debug_mode: bool,+ #[serde(default)]+ pub correct_words: Vec<String>,}fn default_model() -> String {@@ -104,6 +106,7 @@ pub fn get_default_settings() -> AppSettings {selected_language: "auto".to_string(),overlay_position: OverlayPosition::Bottom,debug_mode: false,+ correct_words: Vec::new(),}}diff --git a/src-tauri/src/shortcut.rs b/src-tauri/src/shortcut.rsindex 189f33d5..30eca980 100644--- a/src-tauri/src/shortcut.rs+++ b/src-tauri/src/shortcut.rs@@ -163,6 +163,14 @@ pub fn change_debug_mode_setting(app: AppHandle, enabled: bool) -> Result<(), StOk(())}+#[tauri::command]+pub fn update_correct_words(app: AppHandle, words: Vec<String>) -> Result<(), String> {+ let mut settings = settings::get_settings(&app);+ settings.correct_words = words;+ settings::write_settings(&app, settings);+ Ok(())+}+/// Determine whether a shortcut string contains at least one non-modifier key./// We allow single non-modifier keys (e.g. "f5" or "space") but disallow/// modifier-only combos (e.g. "ctrl" or "ctrl+shift").diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.jsonindex fceaad4f..88674901 100644--- a/src-tauri/tauri.conf.json+++ b/src-tauri/tauri.conf.json@@ -15,9 +15,9 @@{"title": "Handy","width": 540,- "height": 700,+ "height": 750,"minWidth": 540,- "minHeight": 700,+ "minHeight": 750,"resizable": true,"maximizable": false}diff --git a/src/App.tsx b/src/App.tsxindex fbd1a45d..73fe4c34 100644--- a/src/App.tsx+++ b/src/App.tsx@@ -1,12 +1,12 @@import { invoke } from "@tauri-apps/api/core";import { useEffect, useState } from "react";+import { Toaster } from "sonner";import "./App.css";import AccessibilityPermissions from "./components/AccessibilityPermissions";import Footer from "./components/footer";import HandyTextLogo from "./components/icons/HandyTextLogo";import Onboarding from "./components/onboarding";import { Settings } from "./components/settings/Settings";-import { Toaster } from "sonner";function App() {const [showOnboarding, setShowOnboarding] = useState<boolean | null>(null);@@ -35,7 +35,7 @@ function App() {return (<div className="min-h-screen flex flex-col w-full"><div className="flex flex-col items-center pt-6 gap-8 px-4 flex-1">- <HandyTextLogo width={240} />+ <HandyTextLogo width={200} /><Onboarding onModelSelected={handleModelSelected} /></div></div>@@ -46,7 +46,7 @@ function App() {<div className="min-h-screen flex flex-col w-full"><Toaster /><div className="flex flex-col items-center pt-6 gap-8 px-4 flex-1">- <HandyTextLogo width={240} />+ <HandyTextLogo width={200} /><AccessibilityPermissions /><Settings /></div>diff --git a/src/components/settings/CorrectWords.tsx b/src/components/settings/CorrectWords.tsxnew file mode 100644index 00000000..f0908245--- /dev/null+++ b/src/components/settings/CorrectWords.tsx@@ -0,0 +1,113 @@+import React, { useState } from "react";+import { SettingContainer } from "../ui/SettingContainer";+import { useSettings } from "../../hooks/useSettings";++interface CorrectWordsProps {+ descriptionMode?: "inline" | "tooltip";+ grouped?: boolean;+}++export const CorrectWords: React.FC<CorrectWordsProps> = ({+ descriptionMode = "tooltip",+ grouped = false,+}) => {+ const { getSetting, updateSetting, isUpdating } = useSettings();+ const [newWord, setNewWord] = useState("");++ const correctWords = getSetting("correct_words") || [];++ const handleAddWord = () => {+ const trimmedWord = newWord.trim();+ // Don't allow spaces in words+ if (trimmedWord && !trimmedWord.includes(' ') && !correctWords.includes(trimmedWord)) {+ updateSetting("correct_words", [...correctWords, trimmedWord]);+ setNewWord("");+ }+ };++ const handleRemoveWord = (wordToRemove: string) => {+ updateSetting(+ "correct_words",+ correctWords.filter((word) => word !== wordToRemove)+ );+ };++ const handleKeyPress = (e: React.KeyboardEvent) => {+ if (e.key === "Enter") {+ e.preventDefault();+ handleAddWord();+ }+ };++ return (+ <div className={`px-4 p-2 ${grouped ? "" : "rounded-lg border border-mid-gray/20"}`}>+ {/* Title, info button, input and add button all in one line */}+ <div className="flex items-center gap-2">+ <h3 className="text-sm font-medium">Custom Words</h3>+ {descriptionMode === "tooltip" && (+ <div title="Add words that are often misheard or misspelled during transcription. The system will automatically correct similar-sounding words to match your list.">+ <svg+ className="w-4 h-4 text-mid-gray cursor-help hover:text-logo-primary transition-colors duration-200 select-none"+ fill="none"+ stroke="currentColor"+ viewBox="0 0 24 24"+ >+ <path+ strokeLinecap="round"+ strokeLinejoin="round"+ strokeWidth={2}+ d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"+ />+ </svg>+ </div>+ )}+ <input+ type="text"+ value={newWord}+ onChange={(e) => setNewWord(e.target.value)}+ onKeyPress={handleKeyPress}+ placeholder="Add a word"+ className="flex-1 px-2 py-1 text-sm border border-mid-gray/30 rounded focus:outline-none focus:ring-1 focus:ring-logo-primary focus:border-transparent bg-background"+ disabled={isUpdating("correct_words")}+ />+ <button+ onClick={handleAddWord}+ disabled={!newWord.trim() || newWord.includes(' ') || isUpdating("correct_words")}+ className="px-3 py-1 text-xs font-medium text-white bg-logo-primary rounded hover:bg-logo-primary/90 focus:outline-none focus:ring-1 focus:ring-logo-primary disabled:opacity-50 disabled:cursor-not-allowed transition-colors"+ >+ Add+ </button>+ </div>++ {/* Words list - flex wrap layout */}+ {correctWords.length > 0 && (+ <div className="mt-2 flex flex-wrap gap-1">+ {correctWords.map((word) => (+ <button+ key={word}+ onClick={() => handleRemoveWord(word)}+ disabled={isUpdating("correct_words")}+ className="inline-flex items-center gap-1 px-2 py-1 bg-mid-gray/10 rounded text-xs hover:bg-red-100 hover:text-red-700 focus:outline-none disabled:opacity-50 cursor-pointer transition-colors"+ aria-label={`Remove ${word}`}+ >+ <span>{word}</span>+ <svg+ className="w-3 h-3"+ fill="none"+ stroke="currentColor"+ viewBox="0 0 24 24"+ >+ <path+ strokeLinecap="round"+ strokeLinejoin="round"+ strokeWidth={3}+ d="M6 18L18 6M6 6l12 12"+ />+ </svg>+ </button>+ ))}+ </div>+ )}+ </div>+ );+};\ No newline at end of filediff --git a/src/components/settings/Settings.tsx b/src/components/settings/Settings.tsxindex 53c3ab9f..fa530f60 100644--- a/src/components/settings/Settings.tsx+++ b/src/components/settings/Settings.tsx@@ -8,6 +8,7 @@ import { ShowOverlay } from "./ShowOverlay";import { HandyShortcut } from "./HandyShortcut";import { TranslateToEnglish } from "./TranslateToEnglish";import { LanguageSelector } from "./LanguageSelector";+import { CorrectWords } from "./CorrectWords";import { SettingsGroup } from "../ui/SettingsGroup";import { DebugSettings } from "./debug";import { useSettings } from "../../hooks/useSettings";@@ -54,6 +55,7 @@ export const Settings: React.FC = () => {<OutputDeviceSelector descriptionMode="tooltip" grouped={true} /><ShowOverlay descriptionMode="tooltip" grouped={true} /><TranslateToEnglish descriptionMode="tooltip" grouped={true} />+ <CorrectWords descriptionMode="tooltip" grouped={true} /><AlwaysOnMicrophone descriptionMode="tooltip" grouped={true} /></SettingsGroup>diff --git a/src/components/settings/index.ts b/src/components/settings/index.tsindex e8ffd898..cd7dd784 100644--- a/src/components/settings/index.ts+++ b/src/components/settings/index.ts@@ -7,3 +7,4 @@ export { AudioFeedback } from "./AudioFeedback";export { ShowOverlay } from "./ShowOverlay";export { HandyShortcut } from "./HandyShortcut";export { TranslateToEnglish } from "./TranslateToEnglish";+export { CorrectWords } from "./CorrectWords";diff --git a/src/hooks/useSettings.ts b/src/hooks/useSettings.tsindex 17588bc7..5ad26b4b 100644--- a/src/hooks/useSettings.ts+++ b/src/hooks/useSettings.ts@@ -205,6 +205,9 @@ export const useSettings = (): UseSettingsReturn => {case "debug_mode":await invoke("change_debug_mode_setting", { enabled: value });break;+ case "correct_words":+ await invoke("update_correct_words", { words: value });+ break;case "bindings":// Handle bindings separately - they use their own invoke methodsbreak;@@ -251,6 +254,7 @@ export const useSettings = (): UseSettingsReturn => {selected_language: "auto",overlay_position: "bottom",debug_mode: false,+ correct_words: [],};const defaultValue = defaults[key];diff --git a/src/lib/types.ts b/src/lib/types.tsindex dfec7b37..7979b04c 100644--- a/src/lib/types.ts+++ b/src/lib/types.ts@@ -34,6 +34,7 @@ export const SettingsSchema = z.object({selected_language: z.string(),overlay_position: OverlayPositionSchema,debug_mode: z.boolean(),+ correct_words: z.array(z.string()).optional().default([]),});export const BindingResponseSchema = z.object({