💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add an audio playback pipeline, closes #786
davidmezzetti748998d21d2b
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex 8e599a80..b0c41e32 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -54,7 +54,7 @@ jobs:- name: Buildrun: |pip install -U wheel- pip install .[all,dev] fasttext==0.9.2 faiss-cpu==1.8.0 "numpy < 2.0.0" sounddevice speechrecognition webrtcvad-wheels pyaudio+ pip install .[all,dev] fasttext==0.9.2 faiss-cpu==1.8.0 "numpy < 2.0.0"python -c "import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])"python --versionmake data coveragediff --git a/docs/pipeline/audio/audiostream.md b/docs/pipeline/audio/audiostream.mdnew file mode 100644index 00000000..002be107--- /dev/null+++ b/docs/pipeline/audio/audiostream.md@@ -0,0 +1,62 @@+# Audio Stream+++++The Audio Stream pipeline is a threaded pipeline that plays audio segments. This pipeline is designed to run on local machines given that it requires access to write to an output device.++## Example++The following shows a simple example using this pipeline.++```python+from txtai.pipeline import AudioStream++# Create and run pipeline+audio = AudioStream()+audio(data)+```++## Configuration-driven example++Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).++### config.yml+```yaml+# Create pipeline using lower case class name+audiostream:++# Run pipeline with workflow+workflow:+ audiostream:+ tasks:+ - action: audiostream+```++### Run with Workflows++```python+from txtai import Application++# Create and run pipeline with workflow+app = Application("config.yml")+list(app.workflow("audiostream", ["numpy data"]))+```++### Run with API++```bash+CONFIG=config.yml uvicorn "txtai.api:app" &++curl \+ -X POST "http://localhost:8000/workflow" \+ -H "Content-Type: application/json" \+ -d '{"name":"audiostream", "elements":["numpy data"]}'+```++## Methods++Python documentation for the pipeline.++### ::: txtai.pipeline.AudioStream.__init__+### ::: txtai.pipeline.AudioStream.__call__diff --git a/docs/pipeline/index.md b/docs/pipeline/index.mdindex 67dbbc12..4b831a96 100644--- a/docs/pipeline/index.md+++ b/docs/pipeline/index.md@@ -12,6 +12,8 @@ Pipelines are run with Python or configuration. Pipelines can be instantiated inThe following is a list of the default pipelines available in txtai. All pipelines use default models when otherwise not specified. See the [model guide](../models) for the current model recommentations.- Audio+ - [AudioStream](audio/audiostream)+ - [Microphone](audio/microphone)- [TextToSpeech](audio/texttospeech)- [Transcription](audio/transcription)- Data Processingdiff --git a/mkdocs.yml b/mkdocs.ymlindex e281f49c..8846c420 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -74,6 +74,8 @@ nav:- Pipeline:- pipeline/index.md- Audio:+ - Audio Stream: pipeline/audio/audiostream.md+ - Microphone: pipeline/audio/microphone.md- Text To Speech: pipeline/audio/texttospeech.md- Transcription: pipeline/audio/transcription.md- Data:diff --git a/setup.py b/setup.pyindex 5c9994e4..70ea4bff 100644--- a/setup.py+++ b/setup.py@@ -53,7 +53,17 @@ extras["graph"] = ["grand-cypher>=0.6.0", "grand-graph>=0.5.0", "networkx>=2.6.3extras["model"] = ["onnx>=1.11.0", "onnxruntime>=1.11.0"]-extras["pipeline-audio"] = ["onnx>=1.11.0", "onnxruntime>=1.11.0", "soundfile>=0.10.3.post1", "scipy>=1.4.1", "ttstokenizer>=1.0.0"]+extras["pipeline-audio"] = [+ "onnx>=1.11.0",+ "onnxruntime>=1.11.0",+ "pyaudio>=0.2.14",+ "scipy>=1.4.1",+ "speechrecognition>=3.10.4",+ "sounddevice>=0.5.0",+ "soundfile>=0.10.3.post1",+ "ttstokenizer>=1.0.0",+ "webrtcvad-wheels>=2.0.14",+]extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]diff --git a/src/python/txtai/pipeline/audio/audiostream.py b/src/python/txtai/pipeline/audio/audiostream.pynew file mode 100644index 00000000..9feb9135--- /dev/null+++ b/src/python/txtai/pipeline/audio/audiostream.py@@ -0,0 +1,70 @@+"""+AudioStream module+"""++from queue import Queue+from threading import Thread++# Conditional import+try:+ import sounddevice as sd++ SOUNDDEVICE = True+except ImportError:+ SOUNDDEVICE = False++from ..base import Pipeline+++class AudioStream(Pipeline):+ """+ Threaded pipeline that streams audio segments to an output audio device. This pipeline is designed+ to run on local machines given that it requires access to write to an output device.+ """++ COMPLETE = 1++ def __init__(self, rate=22050):+ """+ Creates an AudioStream pipeline.++ Args:+ rate: sample rate of audio segments+ """++ if not SOUNDDEVICE:+ raise ImportError('AudioStream pipeline is not available - install "pipeline" extra to enable')++ # Sampler rate+ self.rate = rate++ self.queue = Queue()+ self.thread = Thread(target=self.play)+ self.thread.start()++ def __call__(self, segment):+ # Convert single element to list+ segments = [segment] if not isinstance(segment, list) else segment++ for x in segments:+ self.queue.put(x)++ # Return single element if single element passed in+ return segment[0] if not isinstance(segment, list) else segment++ def wait(self):+ """+ Waits for all input audio segments to be played.+ """++ self.thread.join()++ def play(self):+ """+ Reads audio segments from queue. This method runs in a separate non-blocking thread.+ """++ audio = self.queue.get()+ while not isinstance(audio, int) or audio != AudioStream.COMPLETE:+ sd.play(audio, self.rate, blocking=True)+ audio = self.queue.get()diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex f1df867e..6d1ba14e 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -45,7 +45,9 @@ class TestOptional(unittest.TestCase):"rich","sklearn.decomposition","sentence_transformers",+ "sounddevice","soundfile",+ "speech_recognition","sqlalchemy","sqlite_vec","tika",@@ -178,12 +180,14 @@ class TestOptional(unittest.TestCase):"""from txtai.pipeline import (+ AudioStream,Caption,HFOnnx,HFTrainer,ImageHash,LiteLLM,LlamaCpp,+ Microphone,MLOnnx,Objects,Segmentation,@@ -194,6 +198,9 @@ class TestOptional(unittest.TestCase):Translation,)+ with self.assertRaises(ImportError):+ AudioStream()+with self.assertRaises(ImportError):Caption()@@ -212,6 +219,9 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):LlamaCpp("TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf")+ with self.assertRaises(ImportError):+ Microphone()+with self.assertRaises(ImportError):MLOnnx()diff --git a/test/python/testpipeline/testaudiostream.py b/test/python/testpipeline/testaudiostream.pynew file mode 100644index 00000000..4a9529e4--- /dev/null+++ b/test/python/testpipeline/testaudiostream.py@@ -0,0 +1,37 @@+"""+AudioStream module tests+"""++import unittest++from unittest.mock import patch++import soundfile as sf++from txtai.pipeline import AudioStream++# pylint: disable = C0411+from utils import Utils+++class TestAudioStream(unittest.TestCase):+ """+ AudioStream tests.+ """++ @patch("sounddevice.play")+ def testAudioStream(self, play):+ """+ Test playing audio+ """++ play.return_value = True++ # Read audio data+ raw, samplerate = sf.read(Utils.PATH + "/Make_huge_profits.wav")++ audio = AudioStream(samplerate)+ self.assertIsNotNone(audio([raw, AudioStream.COMPLETE]))++ # Wait for completion+ audio.wait()