Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

feat(transcription): add word correction with phonetic matching

Vlad Gerasimov003bf23402d7
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index 9203b8fe..6ccf0619 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -1105,7 +1105,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.0",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -2001,6 +2001,7 @@ dependencies = [
"futures-util",
"hound",
"log",
+ "natural",
"once_cell",
"ort-sys",
"rdev",
@@ -2010,6 +2011,7 @@ dependencies = [
"rustfft",
"serde",
"serde_json",
+ "strsim",
"tauri",
"tauri-build",
"tauri-plugin-autostart",
@@ -2962,6 +2964,15 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "natural"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65e6b44f8ddc659cde3555e0140d3441ad26cb03a1410774af1f9a19097c1867"
+dependencies = [
+ "rust-stemmers",
+]
+
[[package]]
name = "ndarray"
version = "0.16.1"
@@ -4365,6 +4376,16 @@ dependencies = [
"realfft",
]
+[[package]]
+name = "rust-stemmers"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
[[package]]
name = "rustc-demangle"
version = "0.1.24"
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index 9b31d845..89f25ac1 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -57,6 +57,8 @@ reqwest = { version = "0.11", features = ["json", "stream"] }
futures-util = "0.3"
tauri-plugin-fs = "2"
rustfft = "6.4.0"
+strsim = "0.11.0"
+natural = "0.5.0"
[dependencies.ort-sys]
version = "=2.0.0-rc.9"
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index e220fac4..9bc07029 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -187,6 +187,7 @@ pub fn run() {
shortcut::change_selected_language_setting,
shortcut::change_overlay_position_setting,
shortcut::change_debug_mode_setting,
+ shortcut::update_correct_words,
shortcut::suspend_binding,
shortcut::resume_binding,
trigger_update_check,
diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs
index bfd3e797..13fc4c20 100644
--- a/src-tauri/src/managers/transcription.rs
+++ b/src-tauri/src/managers/transcription.rs
@@ -1,8 +1,10 @@
use crate::managers::model::ModelManager;
use crate::settings::get_settings;
use anyhow::Result;
+use natural::phonetics::soundex;
use serde::Serialize;
use std::sync::{Arc, Mutex};
+use strsim::levenshtein;
use tauri::{App, AppHandle, Emitter, Manager};
use whisper_rs::install_whisper_log_trampoline;
use whisper_rs::{
@@ -25,6 +27,82 @@ pub struct TranscriptionManager {
current_model_id: Mutex<Option<String>>,
}
+fn correct_words(text: &str, correct_words: &[String]) -> String {
+ if correct_words.is_empty() {
+ return text.to_string();
+ }
+
+ let words: Vec<&str> = text.split_whitespace().collect();
+ let mut corrected_words = Vec::new();
+
+ for word in words {
+ let cleaned_word = word.trim_matches(|c: char| !c.is_alphabetic()).to_lowercase();
+
+ if cleaned_word.is_empty() {
+ corrected_words.push(word.to_string());
+ continue;
+ }
+
+ let mut best_match: Option<&String> = None;
+ let mut best_score = f64::MAX;
+
+ for correct_word in correct_words {
+ let correct_word_lower = correct_word.to_lowercase();
+
+ // Calculate Levenshtein distance (normalized by length)
+ let levenshtein_dist = levenshtein(&cleaned_word, &correct_word_lower);
+ let max_len = cleaned_word.len().max(correct_word_lower.len()) as f64;
+ let levenshtein_score = if max_len > 0.0 { levenshtein_dist as f64 / max_len } else { 1.0 };
+
+ // Calculate phonetic similarity using Soundex
+ let phonetic_match = soundex(&cleaned_word, &correct_word_lower);
+
+ // Combine scores: favor phonetic matches, but also consider string similarity
+ let combined_score = if phonetic_match {
+ levenshtein_score * 0.3 // Give significant boost to phonetic matches
+ } else {
+ levenshtein_score
+ };
+
+ // Accept if the score is good enough (threshold: 0.4 for good matches)
+ if combined_score < 0.4 && combined_score < best_score {
+ best_match = Some(correct_word);
+ best_score = combined_score;
+ }
+ }
+
+ if let Some(replacement) = best_match {
+ // Preserve the original case pattern as much as possible
+ let corrected = if word.chars().all(|c| c.is_uppercase()) {
+ replacement.to_uppercase()
+ } else if word.chars().next().map_or(false, |c| c.is_uppercase()) {
+ let mut chars: Vec<char> = replacement.chars().collect();
+ if let Some(first_char) = chars.get_mut(0) {
+ *first_char = first_char.to_uppercase().next().unwrap_or(*first_char);
+ }
+ chars.into_iter().collect()
+ } else {
+ replacement.clone()
+ };
+
+ // Preserve punctuation from original word
+ let original_suffix: String = word.chars().rev()
+ .take_while(|c| !c.is_alphabetic())
+ .collect::<String>()
+ .chars().rev().collect();
+ let original_prefix: String = word.chars()
+ .take_while(|c| !c.is_alphabetic())
+ .collect();
+
+ corrected_words.push(format!("{}{}{}", original_prefix, corrected, original_suffix));
+ } else {
+ corrected_words.push(word.to_string());
+ }
+ }
+
+ corrected_words.join(" ")
+}
+
impl TranscriptionManager {
pub fn new(app: &App, model_manager: Arc<ModelManager>) -> Result<Self> {
let app_handle = app.app_handle().clone();
@@ -207,6 +285,13 @@ impl TranscriptionManager {
result.push_str(&segment);
}
+ // Apply word correction if correct words are configured
+ let corrected_result = if !settings.correct_words.is_empty() {
+ correct_words(&result, &settings.correct_words)
+ } else {
+ result
+ };
+
let et = std::time::Instant::now();
let translation_note = if settings.translate_to_english {
" (translated)"
@@ -215,6 +300,6 @@ impl TranscriptionManager {
};
println!("\ntook {}ms{}", (et - st).as_millis(), translation_note);
- Ok(result.trim().to_string())
+ Ok(corrected_result.trim().to_string())
}
}
diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs
index 99d1520a..39863d3d 100644
--- a/src-tauri/src/settings.rs
+++ b/src-tauri/src/settings.rs
@@ -42,6 +42,8 @@ pub struct AppSettings {
pub overlay_position: OverlayPosition,
#[serde(default = "default_debug_mode")]
pub debug_mode: bool,
+ #[serde(default)]
+ pub correct_words: Vec<String>,
}
fn default_model() -> String {
@@ -104,6 +106,7 @@ pub fn get_default_settings() -> AppSettings {
selected_language: "auto".to_string(),
overlay_position: OverlayPosition::Bottom,
debug_mode: false,
+ correct_words: Vec::new(),
}
}
diff --git a/src-tauri/src/shortcut.rs b/src-tauri/src/shortcut.rs
index 189f33d5..30eca980 100644
--- a/src-tauri/src/shortcut.rs
+++ b/src-tauri/src/shortcut.rs
@@ -163,6 +163,14 @@ pub fn change_debug_mode_setting(app: AppHandle, enabled: bool) -> Result<(), St
Ok(())
}
+#[tauri::command]
+pub fn update_correct_words(app: AppHandle, words: Vec<String>) -> Result<(), String> {
+ let mut settings = settings::get_settings(&app);
+ settings.correct_words = words;
+ settings::write_settings(&app, settings);
+ Ok(())
+}
+
/// Determine whether a shortcut string contains at least one non-modifier key.
/// We allow single non-modifier keys (e.g. "f5" or "space") but disallow
/// modifier-only combos (e.g. "ctrl" or "ctrl+shift").
diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json
index fceaad4f..88674901 100644
--- a/src-tauri/tauri.conf.json
+++ b/src-tauri/tauri.conf.json
@@ -15,9 +15,9 @@
{
"title": "Handy",
"width": 540,
- "height": 700,
+ "height": 750,
"minWidth": 540,
- "minHeight": 700,
+ "minHeight": 750,
"resizable": true,
"maximizable": false
}
diff --git a/src/App.tsx b/src/App.tsx
index fbd1a45d..73fe4c34 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -1,12 +1,12 @@
import { invoke } from "@tauri-apps/api/core";
import { useEffect, useState } from "react";
+import { Toaster } from "sonner";
import "./App.css";
import AccessibilityPermissions from "./components/AccessibilityPermissions";
import Footer from "./components/footer";
import HandyTextLogo from "./components/icons/HandyTextLogo";
import Onboarding from "./components/onboarding";
import { Settings } from "./components/settings/Settings";
-import { Toaster } from "sonner";
function App() {
const [showOnboarding, setShowOnboarding] = useState<boolean | null>(null);
@@ -35,7 +35,7 @@ function App() {
return (
<div className="min-h-screen flex flex-col w-full">
<div className="flex flex-col items-center pt-6 gap-8 px-4 flex-1">
- <HandyTextLogo width={240} />
+ <HandyTextLogo width={200} />
<Onboarding onModelSelected={handleModelSelected} />
</div>
</div>
@@ -46,7 +46,7 @@ function App() {
<div className="min-h-screen flex flex-col w-full">
<Toaster />
<div className="flex flex-col items-center pt-6 gap-8 px-4 flex-1">
- <HandyTextLogo width={240} />
+ <HandyTextLogo width={200} />
<AccessibilityPermissions />
<Settings />
</div>
diff --git a/src/components/settings/CorrectWords.tsx b/src/components/settings/CorrectWords.tsx
new file mode 100644
index 00000000..f0908245
--- /dev/null
+++ b/src/components/settings/CorrectWords.tsx
@@ -0,0 +1,113 @@
+import React, { useState } from "react";
+import { SettingContainer } from "../ui/SettingContainer";
+import { useSettings } from "../../hooks/useSettings";
+
+interface CorrectWordsProps {
+ descriptionMode?: "inline" | "tooltip";
+ grouped?: boolean;
+}
+
+export const CorrectWords: React.FC<CorrectWordsProps> = ({
+ descriptionMode = "tooltip",
+ grouped = false,
+}) => {
+ const { getSetting, updateSetting, isUpdating } = useSettings();
+ const [newWord, setNewWord] = useState("");
+
+ const correctWords = getSetting("correct_words") || [];
+
+ const handleAddWord = () => {
+ const trimmedWord = newWord.trim();
+ // Don't allow spaces in words
+ if (trimmedWord && !trimmedWord.includes(' ') && !correctWords.includes(trimmedWord)) {
+ updateSetting("correct_words", [...correctWords, trimmedWord]);
+ setNewWord("");
+ }
+ };
+
+ const handleRemoveWord = (wordToRemove: string) => {
+ updateSetting(
+ "correct_words",
+ correctWords.filter((word) => word !== wordToRemove)
+ );
+ };
+
+ const handleKeyPress = (e: React.KeyboardEvent) => {
+ if (e.key === "Enter") {
+ e.preventDefault();
+ handleAddWord();
+ }
+ };
+
+ return (
+ <div className={`px-4 p-2 ${grouped ? "" : "rounded-lg border border-mid-gray/20"}`}>
+ {/* Title, info button, input and add button all in one line */}
+ <div className="flex items-center gap-2">
+ <h3 className="text-sm font-medium">Custom Words</h3>
+ {descriptionMode === "tooltip" && (
+ <div title="Add words that are often misheard or misspelled during transcription. The system will automatically correct similar-sounding words to match your list.">
+ <svg
+ className="w-4 h-4 text-mid-gray cursor-help hover:text-logo-primary transition-colors duration-200 select-none"
+ fill="none"
+ stroke="currentColor"
+ viewBox="0 0 24 24"
+ >
+ <path
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ strokeWidth={2}
+ d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
+ />
+ </svg>
+ </div>
+ )}
+ <input
+ type="text"
+ value={newWord}
+ onChange={(e) => setNewWord(e.target.value)}
+ onKeyPress={handleKeyPress}
+ placeholder="Add a word"
+ className="flex-1 px-2 py-1 text-sm border border-mid-gray/30 rounded focus:outline-none focus:ring-1 focus:ring-logo-primary focus:border-transparent bg-background"
+ disabled={isUpdating("correct_words")}
+ />
+ <button
+ onClick={handleAddWord}
+ disabled={!newWord.trim() || newWord.includes(' ') || isUpdating("correct_words")}
+ className="px-3 py-1 text-xs font-medium text-white bg-logo-primary rounded hover:bg-logo-primary/90 focus:outline-none focus:ring-1 focus:ring-logo-primary disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
+ >
+ Add
+ </button>
+ </div>
+
+ {/* Words list - flex wrap layout */}
+ {correctWords.length > 0 && (
+ <div className="mt-2 flex flex-wrap gap-1">
+ {correctWords.map((word) => (
+ <button
+ key={word}
+ onClick={() => handleRemoveWord(word)}
+ disabled={isUpdating("correct_words")}
+ className="inline-flex items-center gap-1 px-2 py-1 bg-mid-gray/10 rounded text-xs hover:bg-red-100 hover:text-red-700 focus:outline-none disabled:opacity-50 cursor-pointer transition-colors"
+ aria-label={`Remove ${word}`}
+ >
+ <span>{word}</span>
+ <svg
+ className="w-3 h-3"
+ fill="none"
+ stroke="currentColor"
+ viewBox="0 0 24 24"
+ >
+ <path
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ strokeWidth={3}
+ d="M6 18L18 6M6 6l12 12"
+ />
+ </svg>
+ </button>
+ ))}
+ </div>
+ )}
+ </div>
+ );
+};
\ No newline at end of file
diff --git a/src/components/settings/Settings.tsx b/src/components/settings/Settings.tsx
index 53c3ab9f..fa530f60 100644
--- a/src/components/settings/Settings.tsx
+++ b/src/components/settings/Settings.tsx
@@ -8,6 +8,7 @@ import { ShowOverlay } from "./ShowOverlay";
import { HandyShortcut } from "./HandyShortcut";
import { TranslateToEnglish } from "./TranslateToEnglish";
import { LanguageSelector } from "./LanguageSelector";
+import { CorrectWords } from "./CorrectWords";
import { SettingsGroup } from "../ui/SettingsGroup";
import { DebugSettings } from "./debug";
import { useSettings } from "../../hooks/useSettings";
@@ -54,6 +55,7 @@ export const Settings: React.FC = () => {
<OutputDeviceSelector descriptionMode="tooltip" grouped={true} />
<ShowOverlay descriptionMode="tooltip" grouped={true} />
<TranslateToEnglish descriptionMode="tooltip" grouped={true} />
+ <CorrectWords descriptionMode="tooltip" grouped={true} />
<AlwaysOnMicrophone descriptionMode="tooltip" grouped={true} />
</SettingsGroup>
diff --git a/src/components/settings/index.ts b/src/components/settings/index.ts
index e8ffd898..cd7dd784 100644
--- a/src/components/settings/index.ts
+++ b/src/components/settings/index.ts
@@ -7,3 +7,4 @@ export { AudioFeedback } from "./AudioFeedback";
export { ShowOverlay } from "./ShowOverlay";
export { HandyShortcut } from "./HandyShortcut";
export { TranslateToEnglish } from "./TranslateToEnglish";
+export { CorrectWords } from "./CorrectWords";
diff --git a/src/hooks/useSettings.ts b/src/hooks/useSettings.ts
index 17588bc7..5ad26b4b 100644
--- a/src/hooks/useSettings.ts
+++ b/src/hooks/useSettings.ts
@@ -205,6 +205,9 @@ export const useSettings = (): UseSettingsReturn => {
case "debug_mode":
await invoke("change_debug_mode_setting", { enabled: value });
break;
+ case "correct_words":
+ await invoke("update_correct_words", { words: value });
+ break;
case "bindings":
// Handle bindings separately - they use their own invoke methods
break;
@@ -251,6 +254,7 @@ export const useSettings = (): UseSettingsReturn => {
selected_language: "auto",
overlay_position: "bottom",
debug_mode: false,
+ correct_words: [],
};
const defaultValue = defaults[key];
diff --git a/src/lib/types.ts b/src/lib/types.ts
index dfec7b37..7979b04c 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -34,6 +34,7 @@ export const SettingsSchema = z.object({
selected_language: z.string(),
overlay_position: OverlayPositionSchema,
debug_mode: z.boolean(),
+ correct_words: z.array(z.string()).optional().default([]),
});
export const BindingResponseSchema = z.object({