💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor microphone pipeline to remove unnecessary dependencies and improve overall logic #785
davidmezzettiea730ff5ecd3
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex b0c41e32..8daf0eef 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -34,7 +34,7 @@ jobs:java-version: "8"- name: Install dependencies - Linux- run: sudo apt-get update && sudo apt-get install libsndfile1 portaudio19-dev+ run: sudo apt-get update && sudo apt-get install libsndfile1 portaudio19if: matrix.os == 'ubuntu-latest'- name: Install dependencies - macOSdiff --git a/docker/base/Dockerfile b/docker/base/Dockerfileindex ca94fcee..f898deb4 100644--- a/docker/base/Dockerfile+++ b/docker/base/Dockerfile@@ -21,7 +21,7 @@ ENV LANG=C.UTF-8RUN \# Install required packagesapt-get update && \- apt-get -y --no-install-recommends install libgomp1 libsndfile1 portaudio19-dev gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \+ apt-get -y --no-install-recommends install libgomp1 libsndfile1 portaudio19 gcc g++ python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip && \rm -rf /var/lib/apt/lists && \\# Install txtai project and dependenciesdiff --git a/docs/install.md b/docs/install.mdindex fc2f9fd2..ee4c6fed 100644--- a/docs/install.md+++ b/docs/install.md@@ -130,13 +130,13 @@ Additional environment specific prerequisites are below.### Linux-The AudioStream and Microphone pipelines require the [PortAudio](https://people.csail.mit.edu/hubert/pyaudio) system library. The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library. The Transcription pipeline requires the [SoundFile](https://github.com/bastibe/python-soundfile#installation) system library.### macOSOlder versions of Faiss have a runtime dependency on `libomp` for macOS. Run `brew install libomp` in this case.-The AudioStream and Microphone pipelines require the [PortAudio](https://people.csail.mit.edu/hubert/pyaudio) system library.+The AudioStream and Microphone pipelines require the [PortAudio](https://python-sounddevice.readthedocs.io/en/0.5.0/installation.html) system library. Run `brew install portaudio`.### Windowsdiff --git a/docs/pipeline/audio/texttospeech.md b/docs/pipeline/audio/texttospeech.mdindex 0278eea1..31f6a3d4 100644--- a/docs/pipeline/audio/texttospeech.md+++ b/docs/pipeline/audio/texttospeech.md@@ -15,6 +15,16 @@ from txtai.pipeline import TextToSpeech# Create and run pipelinetts = TextToSpeech()tts("Say something here")++# Stream audio - incrementally generates snippets of audio+yield from tts(+ "Say something here. And say something else",+ streaming=True+)++# Generate audio using a speaker id+tts = TextToSpeech("neuml/vctk-vits-onnx")+tts("Say something here", speaker=42)```See the link below for a more detailed example.@@ -27,6 +37,7 @@ This pipeline is backed by ONNX models from the Hugging Face Hub. The following- [ljspeech-jets-onnx](https://huggingface.co/NeuML/ljspeech-jets-onnx)- [ljspeech-vits-onnx](https://huggingface.co/NeuML/ljspeech-vits-onnx)+- [vctk-vits-onnx](https://huggingface.co/NeuML/vctk-vits-onnx)## Configuration-driven examplediff --git a/setup.py b/setup.pyindex 70ea4bff..3bff6496 100644--- a/setup.py+++ b/setup.py@@ -56,9 +56,7 @@ extras["model"] = ["onnx>=1.11.0", "onnxruntime>=1.11.0"]extras["pipeline-audio"] = ["onnx>=1.11.0","onnxruntime>=1.11.0",- "pyaudio>=0.2.14","scipy>=1.4.1",- "speechrecognition>=3.10.4","sounddevice>=0.5.0","soundfile>=0.10.3.post1","ttstokenizer>=1.0.0",diff --git a/src/python/txtai/pipeline/audio/audiostream.py b/src/python/txtai/pipeline/audio/audiostream.pyindex 274a4147..1c564149 100644--- a/src/python/txtai/pipeline/audio/audiostream.py+++ b/src/python/txtai/pipeline/audio/audiostream.py@@ -12,7 +12,7 @@ try:import sounddevice as sdSOUNDDEVICE = True-except ImportError:+except (ImportError, OSError):SOUNDDEVICE = Falsefrom ..base import Pipeline@@ -35,7 +35,7 @@ class AudioStream(Pipeline):"""if not SOUNDDEVICE:- raise ImportError('AudioStream pipeline is not available - install "pipeline" extra to enable')+ raise ImportError("SoundDevice library not installed or portaudio library not found")# Sampler rateself.rate = ratediff --git a/src/python/txtai/pipeline/audio/microphone.py b/src/python/txtai/pipeline/audio/microphone.pyindex bae016b2..bb833c4e 100644--- a/src/python/txtai/pipeline/audio/microphone.py+++ b/src/python/txtai/pipeline/audio/microphone.py@@ -2,106 +2,269 @@Microphone module"""+import logging+import numpy as np# Conditional importtry:- import speech_recognition as sr+ import sounddevice as sdimport webrtcvad- PYAUDIO = True-except ImportError:- PYAUDIO = False+ from scipy.fft import rfft, rfftfreq+ from scipy.signal import butter, sosfilt++ SOUNDDEVICE = True+except (ImportError, OSError):+ SOUNDDEVICE = Falsefrom ..base import Pipeline+# Logging configuration+logger = logging.getLogger(__name__)+class Microphone(Pipeline):"""- Reads input audio from a microphone device. This pipeline is designed to run on local machines given+ Reads input speech from a microphone device. This pipeline is designed to run on local machines giventhat it requires access to read from an input device."""- def __init__(self, rate=16000, vadmode=1, vadframe=30, vadthreshold=0.6):+ def __init__(self, rate=16000, vadmode=3, vadframe=20, vadthreshold=0.6, voicestart=300, voiceend=3400, active=5, pause=8):"""Creates a new Microphone pipeline.Args:- rate: sample rate to record audio in, defaults to 16 kHz- vadmode: aggressiveness of the voice activity detector, defaults to 1- vadframe: voice activity detector frame size in ms, defaults to 30+ rate: sample rate to record audio in, defaults to 16000 (16 kHz)+ vadmode: aggressiveness of the voice activity detector (1 - 3), defaults to 3, which is the most aggressive filter+ vadframe: voice activity detector frame size in ms, defaults to 20vadthreshold: percentage of frames (0.0 - 1.0) that must be voice to be considered speech, defaults to 0.6+ voicestart: starting frequency to use for voice filtering, defaults to 300+ voiceend: ending frequency to use for voice filtering, defaults to 3400+ active: minimum number of active speech chunks to require before considering this speech, defaults to 5+ pause: number of non-speech chunks to keep before considering speech complete, defaults to 8"""- if not PYAUDIO:- raise ImportError('Microphone pipeline is not available - install "pipeline" extra to enable')+ if not SOUNDDEVICE:+ raise ImportError("SoundDevice library not installed or portaudio library not found")++ # Sample rate+ self.rate = rate# Voice activity detectorself.vad = webrtcvad.Vad(vadmode)self.vadframe = vadframeself.vadthreshold = vadthreshold- # Sample rate- self.rate = rate+ # Voice spectrum+ self.voicestart = voicestart+ self.voiceend = voiceend- # Speech recognition config- self.recognizer = sr.Recognizer()+ # Audio chunks counts+ self.active = active+ self.pause = pausedef __call__(self, device=None):- # Read from microphone- with sr.Microphone(sample_rate=self.rate) as source:- # Calibrate microphone- self.recognizer.adjust_for_ambient_noise(source)-- # Wait for speech- audio = None- while audio is None:- audio = self.listen(source)+ # Listen for audio+ audio = self.listen(device[0] if isinstance(device, list) else device)- # Return single element if single element passed in- return (audio, self.rate) if device is None or not isinstance(device, list) else [(audio, self.rate)]+ # Return single element if single element passed in+ return (audio, self.rate) if device is None or not isinstance(device, list) else [(audio, self.rate)]- def listen(self, source):+ def listen(self, device):"""- Listens for audio from source. Returns audio if it passes the voice- activity detector.+ Listens for speech. Detected speech is converted to 32-bit floats for compatibility with+ automatic speech recognition (ASR) pipelines.++ This method blocks until speech is detected.Args:- source: microphone source+ device: input deviceReturns:- audio if present, else None+ audio"""- audio = self.recognizer.listen(source)- if self.detect(audio.frame_data, audio.sample_rate):- # Convert to WAV- data = audio.get_wav_data()+ # Record in 100ms chunks+ chunksize = self.rate // 10++ # Open input stream+ stream = sd.RawInputStream(device=device, samplerate=self.rate, channels=1, blocksize=chunksize, dtype=np.int16)++ # Start the input stream+ stream.start()++ record, speech, nospeech, chunks = True, 0, 0, []+ while record:+ # Read chunk+ chunk, _ = stream.read(chunksize)- # Convert to float32- s16 = np.frombuffer(data, dtype=np.int16, count=len(data) // 2, offset=0)- return s16.astype(np.float32, order="C") / 32768+ # Detect speech using WebRTC VAD for audio chunk+ detect = self.detect(chunk)+ speech = speech + 1 if detect else speech+ nospeech = 0 if detect else nospeech + 1- return None+ # Save chunk, if this is an active stream+ if speech:+ chunks.append(chunk)- def detect(self, audio, rate):+ # Pause limit has been reached, check if this audio should be accepted+ if nospeech >= self.pause:+ logger.debug("Audio detected and being analyzed")+ if speech >= self.active and self.isspeech(chunks[:-nospeech]):+ # Disable recording+ record = False+ else:+ # Reset parameters and keep recording+ logger.debug("Speech not detected")+ speech, nospeech, chunks = 0, 0, []++ # Stop the input stream+ stream.stop()++ # Convert to float32 and return+ audio = np.frombuffer(b"".join(chunks), np.int16)+ return self.float32(audio)++ def isspeech(self, chunks):"""- Voice activity detector.+ Runs an ensemble of Voice Activity Detection (VAD) methods. Returns true if speech is+ detected in the input audio chunks.Args:- audio: input waveform data- rate: sample rate+ chunks: input audio chunks as byte buffers++ Returns:+ True if speech is detected, False otherwise+ """++ # Convert to NumPy array for processing+ audio = np.frombuffer(b"".join(chunks), dtype=np.int16)++ # Ensemble of:+ # - WebRTC VAD with a human voice range butterworth bandpass filter applied to the signal+ # - FFT applied to detect the energy ratio for human voice range vs total range+ return self.detectband(audio) and self.detectenergy(audio)++ def detect(self, buffer):+ """+ Detect speech using the WebRTC Voice Activity Detector (VAD).++ Args:+ buffer: input audio buffer frame as bytesReturns:True if the number of audio frames with audio pass vadthreshold, False otherwise"""- n = int(rate * (self.vadframe / 1000.0) * 2)+ n = int(self.rate * (self.vadframe / 1000.0) * 2)offset = 0detects = []- while offset + n < len(audio):- detects.append(1 if self.vad.is_speech(audio[offset : offset + n], rate) else 0)+ while offset + n <= len(buffer):+ detects.append(1 if self.vad.is_speech(buffer[offset : offset + n], self.rate) else 0)offset += n- return sum(detects) / len(detects) >= self.vadthreshold if detects else 0+ # Calculate detection ratio and return+ ratio = sum(detects) / len(detects) if detects else 0+ logger.debug("DETECT %.4f", ratio)+ return ratio >= self.vadthreshold++ def detectband(self, audio):+ """+ Detects speech using audio data filtered through a butterworth band filter+ with the human voice range.++ Args:+ audio: input audio data as an NumPy array++ Returns:+ True if speech is detected, False otherwise+ """++ # Upsample to float32+ audio = self.float32(audio)++ # Human voice frequency range+ low = self.voicestart / (0.5 * self.rate)+ high = self.voiceend / (0.5 * self.rate)++ # Low and high pass filter using human voice range+ sos = butter(5, Wn=[low, high], btype="band", output="sos")+ audio = sosfilt(sos, audio)++ # Scale back to int16+ audio = self.int16(audio)++ # Pass filtered signal to WebRTC VAD+ return self.detect(audio.tobytes())++ def detectenergy(self, audio):+ """+ Detects speech by comparing the signal energy of the human voice range+ to the overall signal energy.++ Args:+ audio: input audio data as an NumPy array++ Returns:+ True if speech is detected, False otherwise+ """++ # Calculate signal frequency+ frequency = rfftfreq(len(audio), 1.0 / self.rate)+ frequency = frequency[1:]++ # Calculate signal energy using amplitude+ energy = np.abs(rfft(audio))+ energy = energy[1:]+ energy = energy**2++ # Get energy for each frequency+ energyfreq = {}+ for x, freq in enumerate(frequency):+ if abs(freq) not in energyfreq:+ energyfreq[abs(freq)] = energy[x] * 2++ # Sum speech energy+ speechenergy = 0+ for f, e in energyfreq.items():+ if self.voicestart <= f <= self.voiceend:+ speechenergy += e++ # Calculate ratio of speech energy to total energy and return+ ratio = speechenergy / sum(energyfreq.values())+ logger.debug("SPEECH %.4f", ratio)+ return ratio >= self.vadthreshold++ def float32(self, audio):+ """+ Converts an input NumPy array with 16-bit ints to 32-bit floats.++ Args:+ audio: input audio array as 16-bit ints++ Returns:+ audio array as 32-bit floats+ """++ i = np.iinfo(audio.dtype)+ abs_max = 2 ** (i.bits - 1)+ offset = i.min + abs_max+ return (audio.astype(np.float32) - offset) / abs_max++ def int16(self, audio):+ """+ Converts an input NumPy array with 32-bit floats to 16-bit ints.++ Args:+ audio: input audio array as 32-bit floats++ Returns:+ audio array as 16-bit ints+ """++ i = np.iinfo(np.int16)+ absmax = 2 ** (i.bits - 1)+ offset = i.min + absmax+ return (audio * absmax + offset).clip(i.min, i.max).astype(np.int16)diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 6d1ba14e..9394e28e 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -47,7 +47,6 @@ class TestOptional(unittest.TestCase):"sentence_transformers","sounddevice","soundfile",- "speech_recognition","sqlalchemy","sqlite_vec","tika",diff --git a/test/python/testpipeline/testmicrophone.py b/test/python/testpipeline/testmicrophone.pyindex 9997a1d0..2b7a9bcf 100644--- a/test/python/testpipeline/testmicrophone.py+++ b/test/python/testpipeline/testmicrophone.py@@ -6,6 +6,7 @@ import unittestfrom unittest.mock import patch+import numpy as npimport soundfile as sffrom txtai.pipeline import Microphone@@ -19,43 +20,62 @@ class TestMicrophone(unittest.TestCase):Microphone tests."""- @patch("speech_recognition.Recognizer.listen")- @patch("speech_recognition.Recognizer.adjust_for_ambient_noise")- @patch("speech_recognition.Microphone")- # pylint: disable=C0115,C0116,W0613- def testMicrophone(self, microphone, ambient, listen):+ # pylint: disable=C0115,C0116+ @patch("sounddevice.RawInputStream")+ def testMicrophone(self, inputstream):"""Test listening to microphone"""- class Audio:- def __init__(self):- self.frame_data, self.sample_rate = None, None+ class RawInputStream:+ def __init__(self, **kwargs):+ self.args = kwargs- def get_wav_data(self):- return self.frame_data+ # Read audio data+ self.index, self.passes = 0, 0+ audio, self.samplerate = sf.read(Utils.PATH + "/Make_huge_profits.wav")- def speech(device):- # Read audio data- raw, samplerate = sf.read(Utils.PATH + "/Make_huge_profits.wav")+ # Convert data to PCM+ self.audio = self.int16(audio)- audio = Audio()- audio.frame_data, audio.sample_rate = raw, samplerate- return audio+ # Start with random data to test that speech is not detected+ self.data = np.concatenate((self.audio * 50, np.zeros(shape=self.audio.shape, dtype=np.int16)))- def nospeech(device):- audio = Audio()- audio.sample_rate = 16000- audio.frame_data = b"\x00\x00" * int(audio.sample_rate * 30 / 1000)- return audio+ def start(self):+ pass- microphone.return_value.__enter__.return_value = (0, 1)- ambient.return_value = True+ def stop(self):+ pass- pipeline = Microphone()+ def read(self, size):+ # Get chunk+ chunk = self.data[self.index : self.index + size]+ self.index += size++ # Initial pass is random data, 2nd pass is speech data+ if self.index > len(self.data):+ if not self.passes:+ self.index, self.passes = 0, self.passes + 1+ self.data = self.audio+ elif self.index >= len(self.audio) * 10:+ # Break out of loop if speech continues to not be detected+ raise IOError("Data exhausted")++ return chunk, False- listen.side_effect = speech- self.assertIsNotNone(pipeline([1]))+ def int16(self, data):+ i = np.iinfo(np.int16)+ absmax = 2 ** (i.bits - 1)+ offset = i.min + absmax+ return (data * absmax + offset).clip(i.min, i.max).astype(np.int16)++ # Mock input stream+ inputstream.return_value = RawInputStream()++ # Create microphone pipeline and read data+ pipeline = Microphone()+ data, rate = pipeline()- listen.side_effect = nospeech- self.assertIsNone(pipeline.listen(microphone))+ # Validate sample rate and length of data+ self.assertEqual(len(data), 91220)+ self.assertEqual(rate, 16000)