Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

Audio visualizer generally working nicely.

CJ Paisec2fefb9fefd
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index d268f1f6..d9f04608 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -2018,6 +2018,7 @@ dependencies = [
"reqwest 0.11.27",
"rodio",
"rubato",
+ "rustfft",
"serde",
"serde_json",
"tauri",
@@ -4404,9 +4405,9 @@ dependencies = [
[[package]]
name = "rustfft"
-version = "6.2.0"
+version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86"
+checksum = "c6f140db74548f7c9d7cce60912c9ac414e74df5e718dc947d514b051b42f3f4"
dependencies = [
"num-complex",
"num-integer",
@@ -4414,7 +4415,6 @@ dependencies = [
"primal-check",
"strength_reduce",
"transpose",
- "version_check",
]
[[package]]
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index 6407516a..b86d834e 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -52,6 +52,7 @@ rodio = { git = "https://github.com/cjpais/rodio.git" }
reqwest = { version = "0.11", features = ["json", "stream"] }
futures-util = "0.3"
tauri-plugin-fs = "2"
+rustfft = "6.4.0"
[dependencies.ort-sys]
version = "=2.0.0-rc.9"
diff --git a/src-tauri/src/audio_toolkit/audio/mod.rs b/src-tauri/src/audio_toolkit/audio/mod.rs
index e837b8eb..83e307c2 100644
--- a/src-tauri/src/audio_toolkit/audio/mod.rs
+++ b/src-tauri/src/audio_toolkit/audio/mod.rs
@@ -2,7 +2,9 @@
mod device;
mod recorder;
mod resampler;
+mod visualizer;
pub use device::{list_input_devices, list_output_devices, CpalDeviceInfo};
pub use recorder::AudioRecorder;
pub use resampler::FrameResampler;
+pub use visualizer::AudioVisualiser;
diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rs
index 3ec466dd..a3fb1598 100644
--- a/src-tauri/src/audio_toolkit/audio/recorder.rs
+++ b/src-tauri/src/audio_toolkit/audio/recorder.rs
@@ -10,7 +10,7 @@ use cpal::{
};
use crate::audio_toolkit::{
- audio::FrameResampler,
+ audio::{AudioVisualiser, FrameResampler},
constants,
vad::{self, VadFrame},
VoiceActivityDetector,
@@ -27,6 +27,7 @@ pub struct AudioRecorder {
cmd_tx: Option<mpsc::Sender<Cmd>>,
worker_handle: Option<std::thread::JoinHandle<()>>,
vad: Option<Arc<Mutex<Box<dyn vad::VoiceActivityDetector>>>>,
+ level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,
}
impl AudioRecorder {
@@ -36,6 +37,7 @@ impl AudioRecorder {
cmd_tx: None,
worker_handle: None,
vad: None,
+ level_cb: None,
})
}
@@ -44,6 +46,14 @@ impl AudioRecorder {
self
}
+ pub fn with_level_callback<F>(mut self, cb: F) -> Self
+ where
+ F: Fn(Vec<f32>) + Send + Sync + 'static,
+ {
+ self.level_cb = Some(Arc::new(cb));
+ self
+ }
+
pub fn open(&mut self, device: Option<Device>) -> Result<(), Box<dyn std::error::Error>> {
if self.worker_handle.is_some() {
return Ok(()); // already open
@@ -62,6 +72,8 @@ impl AudioRecorder {
let thread_device = device.clone();
let vad = self.vad.clone();
+ // Move the optional level callback into the worker thread
+ let level_cb = self.level_cb.clone();
let worker = std::thread::spawn(move || {
let config = AudioRecorder::get_preferred_config(&thread_device)
@@ -105,7 +117,7 @@ impl AudioRecorder {
stream.play().expect("failed to start stream");
// keep the stream alive while we process samples
- run_consumer(sample_rate, vad, sample_rx, cmd_rx);
+ run_consumer(sample_rate, vad, sample_rx, cmd_rx, level_cb);
// stream is dropped here, after run_consumer returns
});
@@ -215,6 +227,7 @@ fn run_consumer(
vad: Option<Arc<Mutex<Box<dyn vad::VoiceActivityDetector>>>>,
sample_rx: mpsc::Receiver<Vec<f32>>,
cmd_rx: mpsc::Receiver<Cmd>,
+ level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,
) {
let mut frame_resampler = FrameResampler::new(
in_sample_rate as usize,
@@ -225,6 +238,17 @@ fn run_consumer(
let mut processed_samples = Vec::<f32>::new();
let mut recording = false;
+ // ---------- spectrum visualisation setup ---------------------------- //
+ const BUCKETS: usize = 16;
+ const WINDOW_SIZE: usize = 512;
+ let mut visualizer = AudioVisualiser::new(
+ in_sample_rate,
+ WINDOW_SIZE,
+ BUCKETS,
+ 80.0, // vocal_min_hz
+ 4000.0, // vocal_max_hz
+ );
+
fn handle_frame(
samples: &[f32],
recording: bool,
@@ -252,6 +276,14 @@ fn run_consumer(
Err(_) => break, // stream closed
};
+ // ---------- spectrum processing ---------------------------------- //
+ if let Some(buckets) = visualizer.feed(&raw) {
+ if let Some(cb) = &level_cb {
+ cb(buckets);
+ }
+ }
+
+ // ---------- existing pipeline ------------------------------------ //
frame_resampler.push(&raw, &mut |frame: &[f32]| {
handle_frame(frame, recording, &vad, &mut processed_samples)
});
@@ -262,6 +294,7 @@ fn run_consumer(
Cmd::Start => {
processed_samples.clear();
recording = true;
+ visualizer.reset(); // Reset visualization buffer
if let Some(v) = &vad {
v.lock().unwrap().reset();
}
diff --git a/src-tauri/src/audio_toolkit/audio/visualizer.rs b/src-tauri/src/audio_toolkit/audio/visualizer.rs
new file mode 100644
index 00000000..b0ef038e
--- /dev/null
+++ b/src-tauri/src/audio_toolkit/audio/visualizer.rs
@@ -0,0 +1,156 @@
+use rustfft::{num_complex::Complex32, Fft, FftPlanner};
+use std::sync::Arc;
+
+const DB_MIN: f32 = -55.0;
+const DB_MAX: f32 = -8.0;
+const GAIN: f32 = 1.3;
+const CURVE_POWER: f32 = 0.7;
+
+pub struct AudioVisualiser {
+ fft: Arc<dyn Fft<f32>>,
+ window: Vec<f32>,
+ bucket_ranges: Vec<(usize, usize)>,
+ fft_input: Vec<Complex32>,
+ noise_floor: Vec<f32>,
+ buffer: Vec<f32>,
+ window_size: usize,
+ buckets: usize,
+}
+
+impl AudioVisualiser {
+ pub fn new(
+ sample_rate: u32,
+ window_size: usize,
+ buckets: usize,
+ freq_min: f32,
+ freq_max: f32,
+ ) -> Self {
+ let mut planner = FftPlanner::<f32>::new();
+ let fft = planner.plan_fft_forward(window_size);
+
+ // Pre-compute Hann window
+ let window: Vec<f32> = (0..window_size)
+ .map(|i| {
+ 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / window_size as f32).cos())
+ })
+ .collect();
+
+ // Pre-compute bucket frequency ranges
+ let nyquist = sample_rate as f32 / 2.0;
+ let freq_min = freq_min.min(nyquist);
+ let freq_max = freq_max.min(nyquist);
+
+ let mut bucket_ranges = Vec::with_capacity(buckets);
+
+ for b in 0..buckets {
+ // Use logarithmic spacing for better perceptual representation
+ let log_start = (b as f32 / buckets as f32).powi(2);
+ let log_end = ((b + 1) as f32 / buckets as f32).powi(2);
+
+ let start_hz = freq_min + (freq_max - freq_min) * log_start;
+ let end_hz = freq_min + (freq_max - freq_min) * log_end;
+
+ let start_bin = ((start_hz * window_size as f32) / sample_rate as f32) as usize;
+ let mut end_bin = ((end_hz * window_size as f32) / sample_rate as f32) as usize;
+
+ // Ensure each bucket has at least one bin
+ if end_bin <= start_bin {
+ end_bin = start_bin + 1;
+ }
+
+ // Clamp to valid range
+ let start_bin = start_bin.min(window_size / 2);
+ let end_bin = end_bin.min(window_size / 2);
+
+ bucket_ranges.push((start_bin, end_bin));
+ }
+
+ Self {
+ fft,
+ window,
+ bucket_ranges,
+ fft_input: vec![Complex32::new(0.0, 0.0); window_size],
+ noise_floor: vec![-40.0; buckets], // Initialize to reasonable noise floor
+ buffer: Vec::with_capacity(window_size * 2),
+ window_size,
+ buckets,
+ }
+ }
+
+ pub fn feed(&mut self, samples: &[f32]) -> Option<Vec<f32>> {
+ // Add new samples to buffer
+ self.buffer.extend_from_slice(samples);
+
+ // Only process if we have enough samples
+ if self.buffer.len() < self.window_size {
+ return None;
+ }
+
+ // Take the required window of samples
+ let window_samples = &self.buffer[..self.window_size];
+
+ // Remove DC component
+ let mean = window_samples.iter().sum::<f32>() / self.window_size as f32;
+
+ // Apply window function and prepare FFT input
+ for (i, &sample) in window_samples.iter().enumerate() {
+ let windowed_sample = (sample - mean) * self.window[i];
+ self.fft_input[i] = Complex32::new(windowed_sample, 0.0);
+ }
+
+ // Perform FFT
+ self.fft.process(&mut self.fft_input);
+
+ // Compute power spectrum and bucket levels
+ let mut buckets = vec![0.0; self.buckets];
+
+ for (bucket_idx, &(start_bin, end_bin)) in self.bucket_ranges.iter().enumerate() {
+ if start_bin >= end_bin || end_bin > self.fft_input.len() / 2 {
+ continue;
+ }
+
+ // Calculate average power in this frequency range
+ let mut power_sum = 0.0;
+ for bin_idx in start_bin..end_bin {
+ let magnitude = self.fft_input[bin_idx].norm();
+ power_sum += magnitude * magnitude;
+ }
+
+ let avg_power = power_sum / (end_bin - start_bin) as f32;
+
+ // Convert to dB with proper scaling
+ let db = if avg_power > 1e-12 {
+ 20.0 * (avg_power.sqrt() / self.window_size as f32).log10()
+ } else {
+ -80.0 // Very low floor for zero power
+ };
+
+ // Only update noise floor when signal is quiet (below current floor + 10dB)
+ if db < self.noise_floor[bucket_idx] + 10.0 {
+ const NOISE_ALPHA: f32 = 0.001; // Very slow adaptation
+ self.noise_floor[bucket_idx] =
+ NOISE_ALPHA * db + (1.0 - NOISE_ALPHA) * self.noise_floor[bucket_idx];
+ }
+
+ // Map configurable dB range to 0-1 with gain and curve shaping
+ let normalized = ((db - DB_MIN) / (DB_MAX - DB_MIN)).clamp(0.0, 1.0);
+ buckets[bucket_idx] = (normalized * GAIN).powf(CURVE_POWER).clamp(0.0, 1.0);
+ }
+
+ // Apply light smoothing to reduce jitter
+ for i in 1..buckets.len() - 1 {
+ buckets[i] = buckets[i] * 0.7 + buckets[i - 1] * 0.15 + buckets[i + 1] * 0.15;
+ }
+
+ // Clear processed samples from buffer
+ self.buffer.clear();
+
+ Some(buckets)
+ }
+
+ pub fn reset(&mut self) {
+ self.buffer.clear();
+ // Reset noise floor to initial values
+ self.noise_floor.fill(-40.0);
+ }
+}
diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs
index e5c8994c..9bc1ab1f 100644
--- a/src-tauri/src/managers/audio.rs
+++ b/src-tauri/src/managers/audio.rs
@@ -1,9 +1,10 @@
use crate::audio_toolkit::{list_input_devices, vad::SmoothedVad, AudioRecorder, SileroVad};
use crate::settings::get_settings;
+use crate::utils;
use log::{debug, info};
use std::sync::{Arc, Mutex};
use std::time::Instant;
-use tauri::{App, Manager};
+use tauri::{App, Emitter, Manager};
const WHISPER_SAMPLE_RATE: usize = 16000;
@@ -23,13 +24,26 @@ pub enum MicrophoneMode {
/* ──────────────────────────────────────────────────────────────── */
-fn create_audio_recorder(vad_path: &str) -> Result<AudioRecorder, anyhow::Error> {
+fn create_audio_recorder(
+ vad_path: &str,
+ app_handle: &tauri::AppHandle,
+) -> Result<AudioRecorder, anyhow::Error> {
let silero = SileroVad::new(vad_path, 0.3)
.map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?;
let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15, 2);
+
+ // Recorder with VAD plus a spectrum-level callback that forwards updates to
+ // the frontend.
let recorder = AudioRecorder::new()
.map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))?
- .with_vad(Box::new(smoothed_vad));
+ .with_vad(Box::new(smoothed_vad))
+ .with_level_callback({
+ let app_handle = app_handle.clone();
+ move |levels| {
+ utils::emit_levels(&app_handle, &levels);
+ }
+ });
+
Ok(recorder)
}
@@ -97,7 +111,10 @@ impl AudioRecordingManager {
let mut recorder_opt = self.recorder.lock().unwrap();
if recorder_opt.is_none() {
- *recorder_opt = Some(create_audio_recorder(vad_path.to_str().unwrap())?);
+ *recorder_opt = Some(create_audio_recorder(
+ vad_path.to_str().unwrap(),
+ &self.app_handle,
+ )?);
}
// Get the selected device from settings
diff --git a/src-tauri/src/utils.rs b/src-tauri/src/utils.rs
index 0d3d4212..85974659 100644
--- a/src-tauri/src/utils.rs
+++ b/src-tauri/src/utils.rs
@@ -274,3 +274,14 @@ pub fn hide_recording_overlay(app_handle: &AppHandle) {
});
}
}
+
+pub fn emit_levels(app_handle: &AppHandle, levels: &Vec<f32>) {
+ println!("levels {:#?}", levels);
+ // emit levels to main app
+ let _ = app_handle.emit("mic-level", levels);
+
+ // also emit to the recording overlay if it's open
+ if let Some(overlay_window) = app_handle.get_webview_window("recording_overlay") {
+ let _ = overlay_window.emit("mic-level", levels);
+ }
+}
diff --git a/src/overlay/RecordingOverlay.css b/src/overlay/RecordingOverlay.css
index 24f93dc7..80f7e356 100644
--- a/src/overlay/RecordingOverlay.css
+++ b/src/overlay/RecordingOverlay.css
@@ -18,29 +18,19 @@
align-items: end;
justify-content: center;
gap: 3px;
- height: 30px;
+ padding-bottom: 0px;
+ height: 28px;
}
.bar {
- width: 4px;
+ width: 6px;
background: linear-gradient(to top, #faa2ca, #f28cbb);
- border-radius: 2px;
- animation: pulse infinite ease-in-out;
+ max-height: 28px;
+ border-radius: 4px;
+ transition: height 80ms linear;
min-height: 4px;
}
-@keyframes pulse {
- 0%,
- 100% {
- height: 4px;
- opacity: 0.6;
- }
- 50% {
- height: 28px;
- opacity: 1;
- }
-}
-
.recording-overlay.fade-in {
opacity: 1;
}
diff --git a/src/overlay/RecordingOverlay.tsx b/src/overlay/RecordingOverlay.tsx
index 2b6fa024..82971208 100644
--- a/src/overlay/RecordingOverlay.tsx
+++ b/src/overlay/RecordingOverlay.tsx
@@ -1,4 +1,4 @@
-import React, { useEffect, useState } from "react";
+import React, { useEffect, useState, useRef } from "react";
import "./RecordingOverlay.css";
import { listen } from "@tauri-apps/api/event";
@@ -7,6 +7,8 @@ type OverlayState = "recording" | "transcribing";
const RecordingOverlay: React.FC = () => {
const [isVisible, setIsVisible] = useState(false);
const [state, setState] = useState<OverlayState>("recording");
+ const [levels, setLevels] = useState<number[]>(Array(16).fill(0));
+ const smoothedLevelsRef = useRef<number[]>(Array(16).fill(0));
useEffect(() => {
const setupEventListeners = async () => {
@@ -22,10 +24,26 @@ const RecordingOverlay: React.FC = () => {
setIsVisible(false);
});
+ // Listen for mic-level updates
+ const unlistenLevel = await listen<number[]>("mic-level", (event) => {
+ const newLevels = event.payload as number[];
+
+ // Apply smoothing to reduce jitter
+ const smoothed = smoothedLevelsRef.current.map((prev, i) => {
+ const target = newLevels[i] || 0;
+ return prev * 0.7 + target * 0.3; // Smooth transition
+ });
+
+ smoothedLevelsRef.current = smoothed;
+ console.log(smoothed.length);
+ setLevels(smoothed.slice(0, 10));
+ });
+
// Cleanup function
return () => {
unlistenShow();
unlistenHide();
+ unlistenLevel();
};
};
@@ -53,13 +71,14 @@ const RecordingOverlay: React.FC = () => {
/>
{state === "recording" && (
<div className="bars-container">
- {Array.from({ length: 12 }, (_, i) => (
+ {levels.map((v, i) => (
<div
key={i}
className="bar"
style={{
- animationDelay: `${i * 100}ms`,
- animationDuration: `${800 + Math.random() * 400}ms`,
+ height: `${4 + Math.pow(v, 0.7) * 32}px`, // Slight curve for better visual
+ transition: "height 60ms ease-out",
+ opacity: Math.max(0.4, v * 1.7), // Minimum opacity for visibility
}}
/>
))}