Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

Add support for SenseVoice (#745)

CJ Pais4c9c20070986
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index c7c52801..db36fedc 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -7086,10 +7086,11 @@ dependencies = [
[[package]]
name = "transcribe-rs"
-version = "0.2.2"
+version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a15e55d3bd4f74a474b7ff2c87ab416130223db490102c5ad32c6e2227686b0"
+checksum = "2039bf80d6497655ecdef8ebfa10583862fad3cc017077e105bc1c18b7502ff8"
dependencies = [
+ "base64 0.22.1",
"derive_builder",
"env_logger",
"hound",
@@ -7098,6 +7099,7 @@ dependencies = [
"once_cell",
"ort",
"regex",
+ "rustfft",
"serde",
"serde_json",
"thiserror 2.0.17",
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index db7f7473..10742456 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -68,7 +68,7 @@ chrono = "0.4"
rusqlite = { version = "0.37", features = ["bundled"] }
tar = "0.4.44"
flate2 = "1.0"
-transcribe-rs = { version = "0.2.2", features = ["whisper", "parakeet", "moonshine"] }
+transcribe-rs = { version = "0.2.3", features = ["whisper", "parakeet", "moonshine", "sense_voice"] }
handy-keys = "0.1.4"
ferrous-opencc = "0.2.3"
specta = "=2.0.0-rc.22"
diff --git a/src-tauri/src/managers/model.rs b/src-tauri/src/managers/model.rs
index 28fbcfa2..73fbaee1 100644
--- a/src-tauri/src/managers/model.rs
+++ b/src-tauri/src/managers/model.rs
@@ -21,6 +21,7 @@ pub enum EngineType {
Whisper,
Parakeet,
Moonshine,
+ SenseVoice,
}
#[derive(Debug, Clone, Serialize, Deserialize, Type)]
@@ -289,6 +290,37 @@ impl ModelManager {
},
);
+ // SenseVoice supported languages
+ let sense_voice_languages: Vec<String> =
+ vec!["zh", "zh-Hans", "zh-Hant", "en", "yue", "ja", "ko"]
+ .into_iter()
+ .map(String::from)
+ .collect();
+
+ available_models.insert(
+ "sense-voice-int8".to_string(),
+ ModelInfo {
+ id: "sense-voice-int8".to_string(),
+ name: "SenseVoice".to_string(),
+ description: "Very fast. Chinese, English, Japanese, Korean, Cantonese."
+ .to_string(),
+ filename: "sense-voice-int8".to_string(),
+ url: Some("https://blob.handy.computer/sense-voice-int8.tar.gz".to_string()),
+ size_mb: 160,
+ is_downloaded: false,
+ is_downloading: false,
+ partial_size: 0,
+ is_directory: true,
+ engine_type: EngineType::SenseVoice,
+ accuracy_score: 0.65,
+ speed_score: 0.95,
+ supports_translation: false,
+ is_recommended: false,
+ supported_languages: sense_voice_languages,
+ is_custom: false,
+ },
+ );
+
// Auto-discover custom Whisper models (.bin files) in the models directory
if let Err(e) = Self::discover_custom_whisper_models(&models_dir, &mut available_models) {
warn!("Failed to discover custom models: {}", e);
diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs
index 4287533d..0703e1a5 100644
--- a/src-tauri/src/managers/transcription.rs
+++ b/src-tauri/src/managers/transcription.rs
@@ -15,6 +15,10 @@ use transcribe_rs::{
parakeet::{
ParakeetEngine, ParakeetInferenceParams, ParakeetModelParams, TimestampGranularity,
},
+ sense_voice::{
+ Language as SenseVoiceLanguage, SenseVoiceEngine, SenseVoiceInferenceParams,
+ SenseVoiceModelParams,
+ },
whisper::{WhisperEngine, WhisperInferenceParams},
},
TranscriptionEngine,
@@ -32,6 +36,7 @@ enum LoadedEngine {
Whisper(WhisperEngine),
Parakeet(ParakeetEngine),
Moonshine(MoonshineEngine),
+ SenseVoice(SenseVoiceEngine),
}
#[derive(Clone)]
@@ -145,6 +150,7 @@ impl TranscriptionManager {
LoadedEngine::Whisper(ref mut e) => e.unload_model(),
LoadedEngine::Parakeet(ref mut e) => e.unload_model(),
LoadedEngine::Moonshine(ref mut e) => e.unload_model(),
+ LoadedEngine::SenseVoice(ref mut e) => e.unload_model(),
}
}
*engine = None; // Drop the engine to free memory
@@ -284,6 +290,26 @@ impl TranscriptionManager {
})?;
LoadedEngine::Moonshine(engine)
}
+ EngineType::SenseVoice => {
+ let mut engine = SenseVoiceEngine::new();
+ engine
+ .load_model_with_params(&model_path, SenseVoiceModelParams::int8())
+ .map_err(|e| {
+ let error_msg =
+ format!("Failed to load SenseVoice model {}: {}", model_id, e);
+ let _ = self.app_handle.emit(
+ "model-state-changed",
+ ModelStateEvent {
+ event_type: "loading_failed".to_string(),
+ model_id: Some(model_id.to_string()),
+ model_name: Some(model_info.name.clone()),
+ error: Some(error_msg.clone()),
+ },
+ );
+ anyhow::anyhow!(error_msg)
+ })?;
+ LoadedEngine::SenseVoice(engine)
+ }
};
// Update the current engine and model ID
@@ -426,6 +452,23 @@ impl TranscriptionManager {
LoadedEngine::Moonshine(moonshine_engine) => moonshine_engine
.transcribe_samples(audio, None)
.map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e))?,
+ LoadedEngine::SenseVoice(sense_voice_engine) => {
+ let language = match settings.selected_language.as_str() {
+ "zh" | "zh-Hans" | "zh-Hant" => SenseVoiceLanguage::Chinese,
+ "en" => SenseVoiceLanguage::English,
+ "ja" => SenseVoiceLanguage::Japanese,
+ "ko" => SenseVoiceLanguage::Korean,
+ "yue" => SenseVoiceLanguage::Cantonese,
+ _ => SenseVoiceLanguage::Auto,
+ };
+ let params = SenseVoiceInferenceParams {
+ language,
+ use_itn: true,
+ };
+ sense_voice_engine
+ .transcribe_samples(audio, Some(params))
+ .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))?
+ }
}
};
diff --git a/src/bindings.ts b/src/bindings.ts
index cc6a3368..0cbe4398 100644
--- a/src/bindings.ts
+++ b/src/bindings.ts
@@ -700,7 +700,7 @@ export type AudioDevice = { index: string; name: string; is_default: boolean }
export type BindingResponse = { success: boolean; binding: ShortcutBinding | null; error: string | null }
export type ClipboardHandling = "dont_modify" | "copy_to_clipboard"
export type CustomSounds = { start: boolean; stop: boolean }
-export type EngineType = "Whisper" | "Parakeet" | "Moonshine"
+export type EngineType = "Whisper" | "Parakeet" | "Moonshine" | "SenseVoice"
export type HistoryEntry = { id: number; file_name: string; timestamp: number; saved: boolean; title: string; transcription_text: string; post_processed_text: string | null; post_process_prompt: string | null }
/**
* Result of changing keyboard implementation
diff --git a/src/components/settings/LanguageSelector.tsx b/src/components/settings/LanguageSelector.tsx
index 893d98f5..30cba207 100644
--- a/src/components/settings/LanguageSelector.tsx
+++ b/src/components/settings/LanguageSelector.tsx
@@ -8,11 +8,13 @@ import { LANGUAGES } from "../../lib/constants/languages";
interface LanguageSelectorProps {
descriptionMode?: "inline" | "tooltip";
grouped?: boolean;
+ supportedLanguages?: string[];
}
export const LanguageSelector: React.FC<LanguageSelectorProps> = ({
descriptionMode = "tooltip",
grouped = false,
+ supportedLanguages,
}) => {
const { t } = useTranslation();
const { getSetting, updateSetting, resetSetting, isUpdating } = useSettings();
@@ -46,12 +48,21 @@ export const LanguageSelector: React.FC<LanguageSelectorProps> = ({
}
}, [isOpen]);
+ const availableLanguages = useMemo(() => {
+ if (!supportedLanguages || supportedLanguages.length === 0)
+ return LANGUAGES;
+ return LANGUAGES.filter(
+ (lang) =>
+ lang.value === "auto" || supportedLanguages.includes(lang.value),
+ );
+ }, [supportedLanguages]);
+
const filteredLanguages = useMemo(
() =>
- LANGUAGES.filter((language) =>
+ availableLanguages.filter((language) =>
language.label.toLowerCase().includes(searchQuery.toLowerCase()),
),
- [searchQuery],
+ [searchQuery, availableLanguages],
);
const selectedLanguageName =
diff --git a/src/components/settings/general/ModelSettingsCard.tsx b/src/components/settings/general/ModelSettingsCard.tsx
index f91258c2..39b8df45 100644
--- a/src/components/settings/general/ModelSettingsCard.tsx
+++ b/src/components/settings/general/ModelSettingsCard.tsx
@@ -12,8 +12,9 @@ export const ModelSettingsCard: React.FC = () => {
const currentModelInfo = models.find((m: ModelInfo) => m.id === currentModel);
- // Only Whisper models support manual language selection
- const supportsLanguageSelection = currentModelInfo?.engine_type === "Whisper";
+ const supportsLanguageSelection =
+ currentModelInfo?.engine_type === "Whisper" ||
+ currentModelInfo?.engine_type === "SenseVoice";
const supportsTranslation = currentModelInfo?.supports_translation ?? false;
const hasAnySettings = supportsLanguageSelection || supportsTranslation;
@@ -29,7 +30,11 @@ export const ModelSettingsCard: React.FC = () => {
})}
>
{supportsLanguageSelection && (
- <LanguageSelector descriptionMode="tooltip" grouped={true} />
+ <LanguageSelector
+ descriptionMode="tooltip"
+ grouped={true}
+ supportedLanguages={currentModelInfo.supported_languages}
+ />
)}
{supportsTranslation && (
<TranslateToEnglish descriptionMode="tooltip" grouped={true} />
diff --git a/src/lib/constants/languages.ts b/src/lib/constants/languages.ts
index 38920ba8..f4db56bf 100644
--- a/src/lib/constants/languages.ts
+++ b/src/lib/constants/languages.ts
@@ -8,6 +8,7 @@ export const LANGUAGES: Language[] = [
{ value: "en", label: "English" },
{ value: "zh-Hans", label: "Simplified Chinese" },
{ value: "zh-Hant", label: "Traditional Chinese" },
+ { value: "yue", label: "Cantonese" },
{ value: "de", label: "German" },
{ value: "es", label: "Spanish" },
{ value: "ru", label: "Russian" },