Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

Add VRAM Inactivity Timer (#101)

CJ Paisae5f640da535
diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs
index 6db52a55..76b94fd4 100644
--- a/src-tauri/src/commands/mod.rs
+++ b/src-tauri/src/commands/mod.rs
@@ -1,5 +1,6 @@
pub mod audio;
pub mod models;
+pub mod transcription;
use crate::utils::cancel_current_operation;
use tauri::{AppHandle, Manager};
diff --git a/src-tauri/src/commands/transcription.rs b/src-tauri/src/commands/transcription.rs
new file mode 100644
index 00000000..d494b6b9
--- /dev/null
+++ b/src-tauri/src/commands/transcription.rs
@@ -0,0 +1,32 @@
+use crate::managers::transcription::TranscriptionManager;
+use crate::settings::{get_settings, write_settings, ModelUnloadTimeout};
+use tauri::{AppHandle, State};
+
+#[tauri::command]
+pub fn set_model_unload_timeout(app: AppHandle, timeout: ModelUnloadTimeout) {
+ let mut settings = get_settings(&app);
+ settings.model_unload_timeout = timeout;
+ write_settings(&app, settings);
+}
+
+#[tauri::command]
+pub fn get_model_load_status(
+ transcription_manager: State<TranscriptionManager>,
+) -> Result<serde_json::Value, String> {
+ let is_loaded = transcription_manager.is_model_loaded();
+ let current_model = transcription_manager.get_current_model();
+
+ Ok(serde_json::json!({
+ "is_loaded": is_loaded,
+ "current_model": current_model
+ }))
+}
+
+#[tauri::command]
+pub fn unload_model_manually(
+ transcription_manager: State<TranscriptionManager>,
+) -> Result<(), String> {
+ transcription_manager
+ .unload_model()
+ .map_err(|e| format!("Failed to unload model: {}", e))
+}
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 5bf589f3..1a34a714 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -203,7 +203,10 @@ pub fn run() {
commands::audio::get_selected_microphone,
commands::audio::get_available_output_devices,
commands::audio::set_selected_output_device,
- commands::audio::get_selected_output_device
+ commands::audio::get_selected_output_device,
+ commands::transcription::set_model_unload_timeout,
+ commands::transcription::get_model_load_status,
+ commands::transcription::unload_model_manually
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs
index 7de0746a..114fde3e 100644
--- a/src-tauri/src/managers/transcription.rs
+++ b/src-tauri/src/managers/transcription.rs
@@ -1,9 +1,13 @@
use crate::managers::model::ModelManager;
-use crate::settings::get_settings;
+use crate::settings::{get_settings, ModelUnloadTimeout};
use anyhow::Result;
+use log::debug;
use natural::phonetics::soundex;
use serde::Serialize;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
+use std::thread;
+use std::time::{Duration, SystemTime};
use strsim::levenshtein;
use tauri::{App, AppHandle, Emitter, Manager};
use whisper_rs::{
@@ -18,12 +22,16 @@ pub struct ModelStateEvent {
pub error: Option<String>,
}
+#[derive(Clone)]
pub struct TranscriptionManager {
- state: Mutex<Option<WhisperState>>,
- context: Mutex<Option<WhisperContext>>,
+ state: Arc<Mutex<Option<WhisperState>>>,
+ context: Arc<Mutex<Option<WhisperContext>>>,
model_manager: Arc<ModelManager>,
app_handle: AppHandle,
- current_model_id: Mutex<Option<String>>,
+ current_model_id: Arc<Mutex<Option<String>>>,
+ last_activity: Arc<AtomicU64>,
+ shutdown_signal: Arc<AtomicBool>,
+ watcher_handle: Arc<Mutex<Option<thread::JoinHandle<()>>>>,
}
fn apply_custom_words(text: &str, custom_words: &[String], threshold: f64) -> String {
@@ -139,13 +147,81 @@ impl TranscriptionManager {
let app_handle = app.app_handle().clone();
let manager = Self {
- state: Mutex::new(None),
- context: Mutex::new(None),
+ state: Arc::new(Mutex::new(None)),
+ context: Arc::new(Mutex::new(None)),
model_manager,
app_handle: app_handle.clone(),
- current_model_id: Mutex::new(None),
+ current_model_id: Arc::new(Mutex::new(None)),
+ last_activity: Arc::new(AtomicU64::new(
+ SystemTime::now()
+ .duration_since(SystemTime::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64,
+ )),
+ shutdown_signal: Arc::new(AtomicBool::new(false)),
+ watcher_handle: Arc::new(Mutex::new(None)),
};
+ // Start the idle watcher
+ {
+ let app_handle_cloned = app_handle.clone();
+ let manager_cloned = manager.clone();
+ let shutdown_signal = manager.shutdown_signal.clone();
+ let handle = thread::spawn(move || {
+ while !shutdown_signal.load(Ordering::Relaxed) {
+ thread::sleep(Duration::from_secs(10)); // Check every 10 seconds
+
+ // Check shutdown signal again after sleep
+ if shutdown_signal.load(Ordering::Relaxed) {
+ break;
+ }
+
+ let settings = get_settings(&app_handle_cloned);
+ let timeout_seconds = settings.model_unload_timeout.to_seconds();
+
+ if let Some(limit_seconds) = timeout_seconds {
+ // Skip polling-based unloading for immediate timeout since it's handled directly in transcribe()
+ if settings.model_unload_timeout == ModelUnloadTimeout::Immediately {
+ continue;
+ }
+
+ let last = manager_cloned.last_activity.load(Ordering::Relaxed);
+ let now_ms = SystemTime::now()
+ .duration_since(SystemTime::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+
+ if now_ms.saturating_sub(last) > limit_seconds * 1000 {
+ // idle -> unload
+ if manager_cloned.is_model_loaded() {
+ let unload_start = std::time::Instant::now();
+ debug!("Starting to unload model due to inactivity");
+
+ if let Ok(()) = manager_cloned.unload_model() {
+ let _ = app_handle_cloned.emit(
+ "model-state-changed",
+ ModelStateEvent {
+ event_type: "unloaded_due_to_idle".to_string(),
+ model_id: None,
+ model_name: None,
+ error: None,
+ },
+ );
+ let unload_duration = unload_start.elapsed();
+ debug!(
+ "Model unloaded due to inactivity (took {}ms)",
+ unload_duration.as_millis()
+ );
+ }
+ }
+ }
+ }
+ }
+ debug!("Idle watcher thread shutting down gracefully");
+ });
+ *manager.watcher_handle.lock().unwrap() = Some(handle);
+ }
+
// Try to load the default model from settings, but don't fail if no models are available
let settings = get_settings(&app_handle);
let _ = manager.load_model(&settings.selected_model);
@@ -153,7 +229,51 @@ impl TranscriptionManager {
Ok(manager)
}
+ pub fn is_model_loaded(&self) -> bool {
+ let state = self.state.lock().unwrap();
+ state.is_some()
+ }
+
+ pub fn unload_model(&self) -> Result<()> {
+ let unload_start = std::time::Instant::now();
+ debug!("Starting to unload model");
+
+ {
+ let mut state = self.state.lock().unwrap();
+ *state = None; // Dropping state frees GPU/CPU memory
+ }
+ {
+ let mut context = self.context.lock().unwrap();
+ *context = None; // Dropping context frees additional memory
+ }
+ {
+ let mut current_model = self.current_model_id.lock().unwrap();
+ *current_model = None;
+ }
+
+ // Emit unloaded event
+ let _ = self.app_handle.emit(
+ "model-state-changed",
+ ModelStateEvent {
+ event_type: "unloaded_manually".to_string(),
+ model_id: None,
+ model_name: None,
+ error: None,
+ },
+ );
+
+ let unload_duration = unload_start.elapsed();
+ debug!(
+ "Model unloaded manually (took {}ms)",
+ unload_duration.as_millis()
+ );
+ Ok(())
+ }
+
pub fn load_model(&self, model_id: &str) -> Result<()> {
+ let load_start = std::time::Instant::now();
+ debug!("Starting to load model: {}", model_id);
+
// Emit loading started event
let _ = self.app_handle.emit(
"model-state-changed",
@@ -252,7 +372,12 @@ impl TranscriptionManager {
},
);
- println!("Successfully loaded transcription model: {}", model_id);
+ let load_duration = load_start.elapsed();
+ debug!(
+ "Successfully loaded transcription model: {} (took {}ms)",
+ model_id,
+ load_duration.as_millis()
+ );
Ok(())
}
@@ -262,6 +387,15 @@ impl TranscriptionManager {
}
pub fn transcribe(&self, audio: Vec<f32>) -> Result<String> {
+ // Update last activity timestamp
+ self.last_activity.store(
+ SystemTime::now()
+ .duration_since(SystemTime::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64,
+ Ordering::Relaxed,
+ );
+
let st = std::time::Instant::now();
let mut result = String::new();
@@ -272,10 +406,34 @@ impl TranscriptionManager {
return Ok(result);
}
+ // Check if model is loaded, if not try to load it
+ {
+ let state_guard = self.state.lock().unwrap();
+ if state_guard.is_none() {
+ // Model not loaded, try to load the selected model from settings
+ let settings = get_settings(&self.app_handle);
+ println!(
+ "Model not loaded, attempting to load: {}",
+ settings.selected_model
+ );
+
+ // Drop the guard before calling load_model to avoid deadlock
+ drop(state_guard);
+
+ // Try to load the model
+ if let Err(e) = self.load_model(&settings.selected_model) {
+ return Err(anyhow::anyhow!(
+ "Failed to auto-load model '{}': {}. Please check that the model is downloaded and try again.",
+ settings.selected_model, e
+ ));
+ }
+ }
+ }
+
let mut state_guard = self.state.lock().unwrap();
let state = state_guard.as_mut().ok_or_else(|| {
anyhow::anyhow!(
- "No model loaded. Please download and select a model from settings first."
+ "Model failed to load after auto-load attempt. Please check your model settings."
)
})?;
@@ -333,6 +491,34 @@ impl TranscriptionManager {
};
println!("\ntook {}ms{}", (et - st).as_millis(), translation_note);
+ // Check if we should immediately unload the model after transcription
+ if settings.model_unload_timeout == ModelUnloadTimeout::Immediately {
+ println!("⚡ Immediately unloading model after transcription");
+ // Drop the state guard first to avoid deadlock
+ drop(state_guard);
+ if let Err(e) = self.unload_model() {
+ eprintln!("Failed to immediately unload model: {}", e);
+ }
+ }
+
Ok(corrected_result.trim().to_string())
}
}
+
+impl Drop for TranscriptionManager {
+ fn drop(&mut self) {
+ debug!("Shutting down TranscriptionManager");
+
+ // Signal the watcher thread to shutdown
+ self.shutdown_signal.store(true, Ordering::Relaxed);
+
+ // Wait for the thread to finish gracefully
+ if let Some(handle) = self.watcher_handle.lock().unwrap().take() {
+ if let Err(e) = handle.join() {
+ eprintln!("Failed to join idle watcher thread: {:?}", e);
+ } else {
+ debug!("Idle watcher thread joined successfully");
+ }
+ }
+ }
+}
diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs
index 492306ca..2d8b010e 100644
--- a/src-tauri/src/settings.rs
+++ b/src-tauri/src/settings.rs
@@ -20,6 +20,49 @@ pub enum OverlayPosition {
Bottom,
}
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum ModelUnloadTimeout {
+ Never,
+ Immediately,
+ Min2,
+ Min5,
+ Min10,
+ Min15,
+ Hour1,
+ Sec5, // Debug mode only
+}
+
+impl Default for ModelUnloadTimeout {
+ fn default() -> Self {
+ ModelUnloadTimeout::Never
+ }
+}
+
+impl ModelUnloadTimeout {
+ pub fn to_minutes(self) -> Option<u64> {
+ match self {
+ ModelUnloadTimeout::Never => None,
+ ModelUnloadTimeout::Immediately => Some(0), // Special case for immediate unloading
+ ModelUnloadTimeout::Min2 => Some(2),
+ ModelUnloadTimeout::Min5 => Some(5),
+ ModelUnloadTimeout::Min10 => Some(10),
+ ModelUnloadTimeout::Min15 => Some(15),
+ ModelUnloadTimeout::Hour1 => Some(60),
+ ModelUnloadTimeout::Sec5 => Some(0), // Special case for debug - handled separately
+ }
+ }
+
+ pub fn to_seconds(self) -> Option<u64> {
+ match self {
+ ModelUnloadTimeout::Never => None,
+ ModelUnloadTimeout::Immediately => Some(0), // Special case for immediate unloading
+ ModelUnloadTimeout::Sec5 => Some(5),
+ _ => self.to_minutes().map(|m| m * 60),
+ }
+ }
+}
+
/* still handy for composing the initial JSON in the store ------------- */
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct AppSettings {
@@ -44,6 +87,8 @@ pub struct AppSettings {
pub debug_mode: bool,
#[serde(default)]
pub custom_words: Vec<String>,
+ #[serde(default)]
+ pub model_unload_timeout: ModelUnloadTimeout,
#[serde(default = "default_word_correction_threshold")]
pub word_correction_threshold: f64,
}
@@ -113,6 +158,7 @@ pub fn get_default_settings() -> AppSettings {
overlay_position: OverlayPosition::Bottom,
debug_mode: false,
custom_words: Vec::new(),
+ model_unload_timeout: ModelUnloadTimeout::Never,
word_correction_threshold: default_word_correction_threshold(),
}
}
diff --git a/src-tauri/src/shortcut.rs b/src-tauri/src/shortcut.rs
index 5da780e4..9ccb2ad1 100644
--- a/src-tauri/src/shortcut.rs
+++ b/src-tauri/src/shortcut.rs
@@ -1,5 +1,5 @@
use serde::Serialize;
-use tauri::{App, AppHandle, Manager};
+use tauri::{App, AppHandle, Emitter, Manager};
use tauri_plugin_global_shortcut::GlobalShortcutExt;
use tauri_plugin_global_shortcut::{Shortcut, ShortcutState};
@@ -160,6 +160,16 @@ pub fn change_debug_mode_setting(app: AppHandle, enabled: bool) -> Result<(), St
let mut settings = settings::get_settings(&app);
settings.debug_mode = enabled;
settings::write_settings(&app, settings);
+
+ // Emit event to notify frontend of debug mode change
+ let _ = app.emit(
+ "settings-changed",
+ serde_json::json!({
+ "setting": "debug_mode",
+ "value": enabled
+ }),
+ );
+
Ok(())
}
diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json
index 3be32119..cdd986d4 100644
--- a/src-tauri/tauri.conf.json
+++ b/src-tauri/tauri.conf.json
@@ -15,9 +15,9 @@
{
"title": "Handy",
"width": 540,
- "height": 730,
+ "height": 740,
"minWidth": 540,
- "minHeight": 730,
+ "minHeight": 740,
"resizable": true,
"maximizable": false
}
diff --git a/src/components/settings/ModelUnloadTimeout.tsx b/src/components/settings/ModelUnloadTimeout.tsx
new file mode 100644
index 00000000..6df72f3f
--- /dev/null
+++ b/src/components/settings/ModelUnloadTimeout.tsx
@@ -0,0 +1,70 @@
+import React, { useMemo } from "react";
+import { invoke } from "@tauri-apps/api/core";
+import { useSettings } from "../../hooks/useSettings";
+import { ModelUnloadTimeout } from "../../lib/types";
+import { Dropdown } from "../ui/Dropdown";
+import { SettingContainer } from "../ui/SettingContainer";
+
+interface ModelUnloadTimeoutProps {
+ descriptionMode?: "tooltip" | "inline";
+ grouped?: boolean;
+}
+
+const timeoutOptions = [
+ { value: "never" as ModelUnloadTimeout, label: "Never" },
+ { value: "immediately" as ModelUnloadTimeout, label: "Immediately" },
+ { value: "min2" as ModelUnloadTimeout, label: "After 2 minutes" },
+ { value: "min5" as ModelUnloadTimeout, label: "After 5 minutes" },
+ { value: "min10" as ModelUnloadTimeout, label: "After 10 minutes" },
+ { value: "min15" as ModelUnloadTimeout, label: "After 15 minutes" },
+ { value: "hour1" as ModelUnloadTimeout, label: "After 1 hour" },
+];
+
+const debugTimeoutOptions = [
+ ...timeoutOptions,
+ { value: "sec5" as ModelUnloadTimeout, label: "After 5 seconds (Debug)" },
+];
+
+export const ModelUnloadTimeoutSetting: React.FC<ModelUnloadTimeoutProps> = ({
+ descriptionMode = "inline",
+ grouped = false,
+}) => {
+ const { settings, getSetting, updateSetting } = useSettings();
+
+ const handleChange = async (event: React.ChangeEvent<HTMLSelectElement>) => {
+ const newTimeout = event.target.value as ModelUnloadTimeout;
+
+ try {
+ await invoke("set_model_unload_timeout", { timeout: newTimeout });
+ updateSetting("model_unload_timeout", newTimeout);
+ } catch (error) {
+ console.error("Failed to update model unload timeout:", error);
+ }
+ };
+
+ const currentValue = getSetting("model_unload_timeout") ?? "never";
+
+ const options = useMemo(() => {
+ return settings?.debug_mode === true ? debugTimeoutOptions : timeoutOptions;
+ }, [settings]);
+
+ return (
+ <SettingContainer
+ title="Unload Model"
+ description="Automatically free GPU/CPU memory when the model hasn't been used for the specified time"
+ descriptionMode={descriptionMode}
+ grouped={grouped}
+ >
+ <Dropdown
+ options={options}
+ selectedValue={currentValue}
+ onSelect={(value) =>
+ handleChange({
+ target: { value },
+ } as React.ChangeEvent<HTMLSelectElement>)
+ }
+ disabled={false}
+ />
+ </SettingContainer>
+ );
+};
diff --git a/src/components/settings/Settings.tsx b/src/components/settings/Settings.tsx
index 4c6b42e6..db7bb4a0 100644
--- a/src/components/settings/Settings.tsx
+++ b/src/components/settings/Settings.tsx
@@ -9,6 +9,7 @@ import { HandyShortcut } from "./HandyShortcut";
import { TranslateToEnglish } from "./TranslateToEnglish";
import { LanguageSelector } from "./LanguageSelector";
import { CustomWords } from "./CustomWords";
+import { ModelUnloadTimeoutSetting } from "./ModelUnloadTimeout";
import { SettingsGroup } from "../ui/SettingsGroup";
import { WordCorrectionThreshold } from "./debug/WordCorrectionThreshold";
import { AppDataDirectory } from "./AppDataDirectory";
@@ -56,6 +57,7 @@ export const Settings: React.FC = () => {
<OutputDeviceSelector descriptionMode="tooltip" grouped={true} />
<ShowOverlay descriptionMode="tooltip" grouped={true} />
<TranslateToEnglish descriptionMode="tooltip" grouped={true} />
+ <ModelUnloadTimeoutSetting descriptionMode="tooltip" grouped={true} />
<CustomWords descriptionMode="tooltip" grouped />
<AlwaysOnMicrophone descriptionMode="tooltip" grouped={true} />
</SettingsGroup>
diff --git a/src/components/settings/debug/DebugPaths.tsx b/src/components/settings/debug/DebugPaths.tsx
new file mode 100644
index 00000000..6fccca2a
--- /dev/null
+++ b/src/components/settings/debug/DebugPaths.tsx
@@ -0,0 +1,36 @@
+import React from "react";
+import { SettingContainer } from "../../ui/SettingContainer";
+
+interface DebugPathsProps {
+ descriptionMode?: "tooltip" | "inline";
+ grouped?: boolean;
+}
+
+export const DebugPaths: React.FC<DebugPathsProps> = ({
+ descriptionMode = "inline",
+ grouped = false,
+}) => {
+ return (
+ <SettingContainer
+ title="Debug Paths"
+ description="Display internal file paths and directories for debugging purposes"
+ descriptionMode={descriptionMode}
+ grouped={grouped}
+ >
+ <div className="text-sm text-gray-600 space-y-2">
+ <div>
+ <span className="font-medium">App Data:</span>{" "}
+ <span className="font-mono text-xs">%APPDATA%/handy</span>
+ </div>
+ <div>
+ <span className="font-medium">Models:</span>{" "}
+ <span className="font-mono text-xs">%APPDATA%/handy/models</span>
+ </div>
+ <div>
+ <span className="font-medium">Settings:</span>{" "}
+ <span className="font-mono text-xs">%APPDATA%/handy/settings_store.json</span>
+ </div>
+ </div>
+ </SettingContainer>
+ );
+};
diff --git a/src/components/settings/index.ts b/src/components/settings/index.ts
index ff8c3ab7..ec094dea 100644
--- a/src/components/settings/index.ts
+++ b/src/components/settings/index.ts
@@ -9,3 +9,4 @@ export { HandyShortcut } from "./HandyShortcut";
export { TranslateToEnglish } from "./TranslateToEnglish";
export { CustomWords } from "./CustomWords";
export { AppDataDirectory } from "./AppDataDirectory";
+export { ModelUnloadTimeoutSetting } from "./ModelUnloadTimeout";
diff --git a/src/lib/types.ts b/src/lib/types.ts
index ab212eba..8bffb3aa 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -22,6 +22,18 @@ export const AudioDeviceSchema = z.object({
export const OverlayPositionSchema = z.enum(["none", "top", "bottom"]);
export type OverlayPosition = z.infer<typeof OverlayPositionSchema>;
+export const ModelUnloadTimeoutSchema = z.enum([
+ "never",
+ "immediately",
+ "min2",
+ "min5",
+ "min10",
+ "min15",
+ "hour1",
+ "sec5",
+]);
+export type ModelUnloadTimeout = z.infer<typeof ModelUnloadTimeoutSchema>;
+
export const SettingsSchema = z.object({
bindings: ShortcutBindingsMapSchema,
push_to_talk: z.boolean(),
@@ -35,6 +47,7 @@ export const SettingsSchema = z.object({
overlay_position: OverlayPositionSchema,
debug_mode: z.boolean(),
custom_words: z.array(z.string()).optional().default([]),
+ model_unload_timeout: ModelUnloadTimeoutSchema.optional().default("never"),
word_correction_threshold: z.number().optional().default(0.18),
});