π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Upgrade Rubato and test audio resampling contracts
Manav Rathib850ca4815f8
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 175bede711..c4127ac674 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -563,6 +563,39 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "audio-codec-algorithms"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107"++[[package]]+name = "audioadapter"+version = "5.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4"++[[package]]+name = "audioadapter-buffers"+version = "5.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce"+dependencies = [+ "audioadapter",+ "audioadapter-sample",+ "num-traits",+]++[[package]]+name = "audioadapter-sample"+version = "5.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d"+dependencies = [+ "audio-codec-algorithms",+ "num-traits",+]+[[package]]name = "autocfg"version = "1.5.0"@@ -7972,14 +8005,18 @@ dependencies = [[[package]]name = "rubato"-version = "0.16.2"+version = "5.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1"+checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be"dependencies = [+ "audioadapter",+ "audioadapter-buffers","num-complex","num-integer","num-traits","realfft",+ "visibility",+ "windowfunctions",][[package]]@@ -10677,6 +10714,17 @@ version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"+[[package]]+name = "visibility"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "vswhom"version = "0.1.0"@@ -11033,6 +11081,15 @@ dependencies = ["windows-version",]+[[package]]+name = "windowfunctions"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a"+dependencies = [+ "num-traits",+]+[[package]]name = "windows"version = "0.61.3"diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 087b289d1c..715b83daff 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -9,7 +9,7 @@ ente-ensu-crypto.workspace = trueente-image.workspace = truememmap2 = "0.9.9"regex.workspace = true-rubato = { version = "0.16.0", optional = true }+rubato = { version = "5.0.0", optional = true }rusqlite = { workspace = true, default-features = true, optional = true }self_cell = "1.0.0"serde.workspace = truediff --git a/rust/crates/ensu/src/transcription/audio.rs b/rust/crates/ensu/src/transcription/audio.rsindex 5fd996e9fc..f8bb38792d 100644--- a/rust/crates/ensu/src/transcription/audio.rs+++ b/rust/crates/ensu/src/transcription/audio.rs@@ -1,7 +1,8 @@use std::path::Path;use std::time::Duration;-use rubato::{FftFixedIn, Resampler};+use rubato::audioadapter_buffers::direct::InterleavedSlice;+use rubato::{Fft, FixedSync, Resampler, WindowFunction};use transcribe_rs::vad::{SileroVad, SmoothedVad, Vad};use crate::transcription::{Result, TranscriptionError};@@ -96,7 +97,7 @@ fn extract_speech(}struct FrameResampler {- resampler: Option<FftFixedIn<f32>>,+ resampler: Option<Fft<f32>>,chunk_in: usize,in_buf: Vec<f32>,frame_samples: usize,@@ -115,12 +116,14 @@ impl FrameResampler {let resampler = if in_hz == out_hz {None} else {- Some(FftFixedIn::<f32>::new(+ Some(Fft::<f32>::new_custom(in_hz,out_hz,RESAMPLER_CHUNK_SIZE,1,1,+ WindowFunction::BlackmanHarris2,+ FixedSync::Input,)?)};@@ -147,9 +150,10 @@ impl FrameResampler {if self.in_buf.len() == self.chunk_in {if let Some(resampler) = self.resampler.as_mut()- && let Ok(out) = resampler.process(&[&self.in_buf[..]], None)+ && let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)+ && let Ok(out) = resampler.process(&input, None){- self.emit_frames(&out[0], &mut emit);+ self.emit_frames(&out.take_data(), &mut emit);}self.in_buf.clear();}@@ -161,8 +165,10 @@ impl FrameResampler {&& !self.in_buf.is_empty(){self.in_buf.resize(self.chunk_in, 0.0);- if let Ok(out) = resampler.process(&[&self.in_buf[..]], None) {- self.emit_frames(&out[0], &mut emit);+ if let Ok(input) = InterleavedSlice::new(&self.in_buf, 1, self.chunk_in)+ && let Ok(out) = resampler.process(&input, None)+ {+ self.emit_frames(&out.take_data(), &mut emit);}}@@ -187,3 +193,85 @@ impl FrameResampler {}}}++#[cfg(test)]+mod tests {+ use std::f32::consts::TAU;++ use super::*;++ fn resample(input: &[f32], sample_rate: usize, push_size: usize) -> Vec<f32> {+ let mut resampler =+ FrameResampler::new(sample_rate, 16_000, Duration::from_millis(30)).unwrap();+ let mut output = Vec::new();+ let mut emit = |frame: &[f32]| {+ assert_eq!(frame.len(), 480);+ output.extend_from_slice(frame);+ };+ for chunk in input.chunks(push_size) {+ resampler.push(chunk, &mut emit);+ }+ resampler.finish(emit);+ output+ }++ #[test]+ fn native_rate_preserves_samples_and_pads_the_last_frame() {+ let input: Vec<_> = (0..961).map(|i| i as f32 / 1000.0 - 0.5).collect();+ let output = resample(&input, 16_000, 137);++ assert_eq!(output.len(), 1440);+ assert_eq!(&output[..input.len()], input);+ assert!(output[input.len()..].iter().all(|&sample| sample == 0.0));+ }++ #[test]+ fn resampling_is_independent_of_push_boundaries() {+ for sample_rate in [44_100, 48_000] {+ let input: Vec<_> = (0..sample_rate / 4 + 19)+ .map(|i| (TAU * 1000.0 * i as f32 / sample_rate as f32).sin())+ .collect();+ let expected = resample(&input, sample_rate, input.len());+ assert!(!expected.is_empty());++ for push_size in [1, 257, 1023, 1024, 1025] {+ assert_eq!(+ resample(&input, sample_rate, push_size),+ expected,+ "{sample_rate} Hz, {push_size} samples per push"+ );+ }+ }+ }++ #[test]+ fn downsampling_preserves_speech_frequencies_and_rejects_aliasing() {+ fn amplitude(samples: &[f32], frequency: f32) -> f32 {+ let (sin, cos) =+ samples+ .iter()+ .enumerate()+ .fold((0.0, 0.0), |(sin, cos), (i, &sample)| {+ let phase = TAU * frequency * i as f32 / 16_000.0;+ (sin + sample * phase.sin(), cos + sample * phase.cos())+ });+ 2.0 * sin.hypot(cos) / samples.len() as f32+ }++ for sample_rate in [44_100, 48_000] {+ let input: Vec<_> = (0..sample_rate)+ .map(|i| {+ let phase = TAU * i as f32 / sample_rate as f32;+ 0.5 * (1000.0 * phase).sin() + 0.5 * (12_000.0 * phase).sin()+ })+ .collect();+ let output = resample(&input, sample_rate, 257);+ let settled = &output[1600..9600];+ let speech = amplitude(settled, 1000.0);+ let alias = amplitude(settled, 4000.0);++ assert!((speech - 0.5).abs() < 0.01, "{sample_rate} Hz: {speech}");+ assert!(alias < 0.01, "{sample_rate} Hz: {alias}");+ }+ }+}