A free, open source, and extensible speech-to-text application that works completely offline.
Audio visualizer generally working nicely.
CJ Paisec2fefb9fefd
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lockindex d268f1f6..d9f04608 100644--- a/src-tauri/Cargo.lock+++ b/src-tauri/Cargo.lock@@ -2018,6 +2018,7 @@ dependencies = ["reqwest 0.11.27","rodio","rubato",+ "rustfft","serde","serde_json","tauri",@@ -4404,9 +4405,9 @@ dependencies = [[[package]]name = "rustfft"-version = "6.2.0"+version = "6.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86"+checksum = "c6f140db74548f7c9d7cce60912c9ac414e74df5e718dc947d514b051b42f3f4"dependencies = ["num-complex","num-integer",@@ -4414,7 +4415,6 @@ dependencies = ["primal-check","strength_reduce","transpose",- "version_check",][[package]]diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.tomlindex 6407516a..b86d834e 100644--- a/src-tauri/Cargo.toml+++ b/src-tauri/Cargo.toml@@ -52,6 +52,7 @@ rodio = { git = "https://github.com/cjpais/rodio.git" }reqwest = { version = "0.11", features = ["json", "stream"] }futures-util = "0.3"tauri-plugin-fs = "2"+rustfft = "6.4.0"[dependencies.ort-sys]version = "=2.0.0-rc.9"diff --git a/src-tauri/src/audio_toolkit/audio/mod.rs b/src-tauri/src/audio_toolkit/audio/mod.rsindex e837b8eb..83e307c2 100644--- a/src-tauri/src/audio_toolkit/audio/mod.rs+++ b/src-tauri/src/audio_toolkit/audio/mod.rs@@ -2,7 +2,9 @@mod device;mod recorder;mod resampler;+mod visualizer;pub use device::{list_input_devices, list_output_devices, CpalDeviceInfo};pub use recorder::AudioRecorder;pub use resampler::FrameResampler;+pub use visualizer::AudioVisualiser;diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rsindex 3ec466dd..a3fb1598 100644--- a/src-tauri/src/audio_toolkit/audio/recorder.rs+++ b/src-tauri/src/audio_toolkit/audio/recorder.rs@@ -10,7 +10,7 @@ use cpal::{};use crate::audio_toolkit::{- audio::FrameResampler,+ audio::{AudioVisualiser, FrameResampler},constants,vad::{self, VadFrame},VoiceActivityDetector,@@ -27,6 +27,7 @@ pub struct AudioRecorder {cmd_tx: Option<mpsc::Sender<Cmd>>,worker_handle: Option<std::thread::JoinHandle<()>>,vad: Option<Arc<Mutex<Box<dyn vad::VoiceActivityDetector>>>>,+ level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,}impl AudioRecorder {@@ -36,6 +37,7 @@ impl AudioRecorder {cmd_tx: None,worker_handle: None,vad: None,+ level_cb: None,})}@@ -44,6 +46,14 @@ impl AudioRecorder {self}+ pub fn with_level_callback<F>(mut self, cb: F) -> Self+ where+ F: Fn(Vec<f32>) + Send + Sync + 'static,+ {+ self.level_cb = Some(Arc::new(cb));+ self+ }+pub fn open(&mut self, device: Option<Device>) -> Result<(), Box<dyn std::error::Error>> {if self.worker_handle.is_some() {return Ok(()); // already open@@ -62,6 +72,8 @@ impl AudioRecorder {let thread_device = device.clone();let vad = self.vad.clone();+ // Move the optional level callback into the worker thread+ let level_cb = self.level_cb.clone();let worker = std::thread::spawn(move || {let config = AudioRecorder::get_preferred_config(&thread_device)@@ -105,7 +117,7 @@ impl AudioRecorder {stream.play().expect("failed to start stream");// keep the stream alive while we process samples- run_consumer(sample_rate, vad, sample_rx, cmd_rx);+ run_consumer(sample_rate, vad, sample_rx, cmd_rx, level_cb);// stream is dropped here, after run_consumer returns});@@ -215,6 +227,7 @@ fn run_consumer(vad: Option<Arc<Mutex<Box<dyn vad::VoiceActivityDetector>>>>,sample_rx: mpsc::Receiver<Vec<f32>>,cmd_rx: mpsc::Receiver<Cmd>,+ level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,) {let mut frame_resampler = FrameResampler::new(in_sample_rate as usize,@@ -225,6 +238,17 @@ fn run_consumer(let mut processed_samples = Vec::<f32>::new();let mut recording = false;+ // ---------- spectrum visualisation setup ---------------------------- //+ const BUCKETS: usize = 16;+ const WINDOW_SIZE: usize = 512;+ let mut visualizer = AudioVisualiser::new(+ in_sample_rate,+ WINDOW_SIZE,+ BUCKETS,+ 80.0, // vocal_min_hz+ 4000.0, // vocal_max_hz+ );+fn handle_frame(samples: &[f32],recording: bool,@@ -252,6 +276,14 @@ fn run_consumer(Err(_) => break, // stream closed};+ // ---------- spectrum processing ---------------------------------- //+ if let Some(buckets) = visualizer.feed(&raw) {+ if let Some(cb) = &level_cb {+ cb(buckets);+ }+ }++ // ---------- existing pipeline ------------------------------------ //frame_resampler.push(&raw, &mut |frame: &[f32]| {handle_frame(frame, recording, &vad, &mut processed_samples)});@@ -262,6 +294,7 @@ fn run_consumer(Cmd::Start => {processed_samples.clear();recording = true;+ visualizer.reset(); // Reset visualization bufferif let Some(v) = &vad {v.lock().unwrap().reset();}diff --git a/src-tauri/src/audio_toolkit/audio/visualizer.rs b/src-tauri/src/audio_toolkit/audio/visualizer.rsnew file mode 100644index 00000000..b0ef038e--- /dev/null+++ b/src-tauri/src/audio_toolkit/audio/visualizer.rs@@ -0,0 +1,156 @@+use rustfft::{num_complex::Complex32, Fft, FftPlanner};+use std::sync::Arc;++const DB_MIN: f32 = -55.0;+const DB_MAX: f32 = -8.0;+const GAIN: f32 = 1.3;+const CURVE_POWER: f32 = 0.7;++pub struct AudioVisualiser {+ fft: Arc<dyn Fft<f32>>,+ window: Vec<f32>,+ bucket_ranges: Vec<(usize, usize)>,+ fft_input: Vec<Complex32>,+ noise_floor: Vec<f32>,+ buffer: Vec<f32>,+ window_size: usize,+ buckets: usize,+}++impl AudioVisualiser {+ pub fn new(+ sample_rate: u32,+ window_size: usize,+ buckets: usize,+ freq_min: f32,+ freq_max: f32,+ ) -> Self {+ let mut planner = FftPlanner::<f32>::new();+ let fft = planner.plan_fft_forward(window_size);++ // Pre-compute Hann window+ let window: Vec<f32> = (0..window_size)+ .map(|i| {+ 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / window_size as f32).cos())+ })+ .collect();++ // Pre-compute bucket frequency ranges+ let nyquist = sample_rate as f32 / 2.0;+ let freq_min = freq_min.min(nyquist);+ let freq_max = freq_max.min(nyquist);++ let mut bucket_ranges = Vec::with_capacity(buckets);++ for b in 0..buckets {+ // Use logarithmic spacing for better perceptual representation+ let log_start = (b as f32 / buckets as f32).powi(2);+ let log_end = ((b + 1) as f32 / buckets as f32).powi(2);++ let start_hz = freq_min + (freq_max - freq_min) * log_start;+ let end_hz = freq_min + (freq_max - freq_min) * log_end;++ let start_bin = ((start_hz * window_size as f32) / sample_rate as f32) as usize;+ let mut end_bin = ((end_hz * window_size as f32) / sample_rate as f32) as usize;++ // Ensure each bucket has at least one bin+ if end_bin <= start_bin {+ end_bin = start_bin + 1;+ }++ // Clamp to valid range+ let start_bin = start_bin.min(window_size / 2);+ let end_bin = end_bin.min(window_size / 2);++ bucket_ranges.push((start_bin, end_bin));+ }++ Self {+ fft,+ window,+ bucket_ranges,+ fft_input: vec![Complex32::new(0.0, 0.0); window_size],+ noise_floor: vec![-40.0; buckets], // Initialize to reasonable noise floor+ buffer: Vec::with_capacity(window_size * 2),+ window_size,+ buckets,+ }+ }++ pub fn feed(&mut self, samples: &[f32]) -> Option<Vec<f32>> {+ // Add new samples to buffer+ self.buffer.extend_from_slice(samples);++ // Only process if we have enough samples+ if self.buffer.len() < self.window_size {+ return None;+ }++ // Take the required window of samples+ let window_samples = &self.buffer[..self.window_size];++ // Remove DC component+ let mean = window_samples.iter().sum::<f32>() / self.window_size as f32;++ // Apply window function and prepare FFT input+ for (i, &sample) in window_samples.iter().enumerate() {+ let windowed_sample = (sample - mean) * self.window[i];+ self.fft_input[i] = Complex32::new(windowed_sample, 0.0);+ }++ // Perform FFT+ self.fft.process(&mut self.fft_input);++ // Compute power spectrum and bucket levels+ let mut buckets = vec![0.0; self.buckets];++ for (bucket_idx, &(start_bin, end_bin)) in self.bucket_ranges.iter().enumerate() {+ if start_bin >= end_bin || end_bin > self.fft_input.len() / 2 {+ continue;+ }++ // Calculate average power in this frequency range+ let mut power_sum = 0.0;+ for bin_idx in start_bin..end_bin {+ let magnitude = self.fft_input[bin_idx].norm();+ power_sum += magnitude * magnitude;+ }++ let avg_power = power_sum / (end_bin - start_bin) as f32;++ // Convert to dB with proper scaling+ let db = if avg_power > 1e-12 {+ 20.0 * (avg_power.sqrt() / self.window_size as f32).log10()+ } else {+ -80.0 // Very low floor for zero power+ };++ // Only update noise floor when signal is quiet (below current floor + 10dB)+ if db < self.noise_floor[bucket_idx] + 10.0 {+ const NOISE_ALPHA: f32 = 0.001; // Very slow adaptation+ self.noise_floor[bucket_idx] =+ NOISE_ALPHA * db + (1.0 - NOISE_ALPHA) * self.noise_floor[bucket_idx];+ }++ // Map configurable dB range to 0-1 with gain and curve shaping+ let normalized = ((db - DB_MIN) / (DB_MAX - DB_MIN)).clamp(0.0, 1.0);+ buckets[bucket_idx] = (normalized * GAIN).powf(CURVE_POWER).clamp(0.0, 1.0);+ }++ // Apply light smoothing to reduce jitter+ for i in 1..buckets.len() - 1 {+ buckets[i] = buckets[i] * 0.7 + buckets[i - 1] * 0.15 + buckets[i + 1] * 0.15;+ }++ // Clear processed samples from buffer+ self.buffer.clear();++ Some(buckets)+ }++ pub fn reset(&mut self) {+ self.buffer.clear();+ // Reset noise floor to initial values+ self.noise_floor.fill(-40.0);+ }+}diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rsindex e5c8994c..9bc1ab1f 100644--- a/src-tauri/src/managers/audio.rs+++ b/src-tauri/src/managers/audio.rs@@ -1,9 +1,10 @@use crate::audio_toolkit::{list_input_devices, vad::SmoothedVad, AudioRecorder, SileroVad};use crate::settings::get_settings;+use crate::utils;use log::{debug, info};use std::sync::{Arc, Mutex};use std::time::Instant;-use tauri::{App, Manager};+use tauri::{App, Emitter, Manager};const WHISPER_SAMPLE_RATE: usize = 16000;@@ -23,13 +24,26 @@ pub enum MicrophoneMode {/* ──────────────────────────────────────────────────────────────── */-fn create_audio_recorder(vad_path: &str) -> Result<AudioRecorder, anyhow::Error> {+fn create_audio_recorder(+ vad_path: &str,+ app_handle: &tauri::AppHandle,+) -> Result<AudioRecorder, anyhow::Error> {let silero = SileroVad::new(vad_path, 0.3).map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?;let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15, 2);++ // Recorder with VAD plus a spectrum-level callback that forwards updates to+ // the frontend.let recorder = AudioRecorder::new().map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))?- .with_vad(Box::new(smoothed_vad));+ .with_vad(Box::new(smoothed_vad))+ .with_level_callback({+ let app_handle = app_handle.clone();+ move |levels| {+ utils::emit_levels(&app_handle, &levels);+ }+ });+Ok(recorder)}@@ -97,7 +111,10 @@ impl AudioRecordingManager {let mut recorder_opt = self.recorder.lock().unwrap();if recorder_opt.is_none() {- *recorder_opt = Some(create_audio_recorder(vad_path.to_str().unwrap())?);+ *recorder_opt = Some(create_audio_recorder(+ vad_path.to_str().unwrap(),+ &self.app_handle,+ )?);}// Get the selected device from settingsdiff --git a/src-tauri/src/utils.rs b/src-tauri/src/utils.rsindex 0d3d4212..85974659 100644--- a/src-tauri/src/utils.rs+++ b/src-tauri/src/utils.rs@@ -274,3 +274,14 @@ pub fn hide_recording_overlay(app_handle: &AppHandle) {});}}++pub fn emit_levels(app_handle: &AppHandle, levels: &Vec<f32>) {+ println!("levels {:#?}", levels);+ // emit levels to main app+ let _ = app_handle.emit("mic-level", levels);++ // also emit to the recording overlay if it's open+ if let Some(overlay_window) = app_handle.get_webview_window("recording_overlay") {+ let _ = overlay_window.emit("mic-level", levels);+ }+}diff --git a/src/overlay/RecordingOverlay.css b/src/overlay/RecordingOverlay.cssindex 24f93dc7..80f7e356 100644--- a/src/overlay/RecordingOverlay.css+++ b/src/overlay/RecordingOverlay.css@@ -18,29 +18,19 @@align-items: end;justify-content: center;gap: 3px;- height: 30px;+ padding-bottom: 0px;+ height: 28px;}.bar {- width: 4px;+ width: 6px;background: linear-gradient(to top, #faa2ca, #f28cbb);- border-radius: 2px;- animation: pulse infinite ease-in-out;+ max-height: 28px;+ border-radius: 4px;+ transition: height 80ms linear;min-height: 4px;}-@keyframes pulse {- 0%,- 100% {- height: 4px;- opacity: 0.6;- }- 50% {- height: 28px;- opacity: 1;- }-}-.recording-overlay.fade-in {opacity: 1;}diff --git a/src/overlay/RecordingOverlay.tsx b/src/overlay/RecordingOverlay.tsxindex 2b6fa024..82971208 100644--- a/src/overlay/RecordingOverlay.tsx+++ b/src/overlay/RecordingOverlay.tsx@@ -1,4 +1,4 @@-import React, { useEffect, useState } from "react";+import React, { useEffect, useState, useRef } from "react";import "./RecordingOverlay.css";import { listen } from "@tauri-apps/api/event";@@ -7,6 +7,8 @@ type OverlayState = "recording" | "transcribing";const RecordingOverlay: React.FC = () => {const [isVisible, setIsVisible] = useState(false);const [state, setState] = useState<OverlayState>("recording");+ const [levels, setLevels] = useState<number[]>(Array(16).fill(0));+ const smoothedLevelsRef = useRef<number[]>(Array(16).fill(0));useEffect(() => {const setupEventListeners = async () => {@@ -22,10 +24,26 @@ const RecordingOverlay: React.FC = () => {setIsVisible(false);});+ // Listen for mic-level updates+ const unlistenLevel = await listen<number[]>("mic-level", (event) => {+ const newLevels = event.payload as number[];++ // Apply smoothing to reduce jitter+ const smoothed = smoothedLevelsRef.current.map((prev, i) => {+ const target = newLevels[i] || 0;+ return prev * 0.7 + target * 0.3; // Smooth transition+ });++ smoothedLevelsRef.current = smoothed;+ console.log(smoothed.length);+ setLevels(smoothed.slice(0, 10));+ });+// Cleanup functionreturn () => {unlistenShow();unlistenHide();+ unlistenLevel();};};@@ -53,13 +71,14 @@ const RecordingOverlay: React.FC = () => {/>{state === "recording" && (<div className="bars-container">- {Array.from({ length: 12 }, (_, i) => (+ {levels.map((v, i) => (<divkey={i}className="bar"style={{- animationDelay: `${i * 100}ms`,- animationDuration: `${800 + Math.random() * 400}ms`,+ height: `${4 + Math.pow(v, 0.7) * 32}px`, // Slight curve for better visual+ transition: "height 60ms ease-out",+ opacity: Math.max(0.4, v * 1.7), // Minimum opacity for visibility}}/>))}