Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add an audio playback pipeline, closes #786

davidmezzetti748998d21d2b
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8e599a80..b0c41e32 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -54,7 +54,7 @@ jobs:
- name: Build
run: |
pip install -U wheel
- pip install .[all,dev] fasttext==0.9.2 faiss-cpu==1.8.0 "numpy < 2.0.0" sounddevice speechrecognition webrtcvad-wheels pyaudio
+ pip install .[all,dev] fasttext==0.9.2 faiss-cpu==1.8.0 "numpy < 2.0.0"
python -c "import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])"
python --version
make data coverage
diff --git a/docs/pipeline/audio/audiostream.md b/docs/pipeline/audio/audiostream.md
new file mode 100644
index 00000000..002be107
--- /dev/null
+++ b/docs/pipeline/audio/audiostream.md
@@ -0,0 +1,62 @@
+# Audio Stream
+
+![pipeline](../../images/pipeline.png#only-light)
+![pipeline](../../images/pipeline-dark.png#only-dark)
+
+The Audio Stream pipeline is a threaded pipeline that plays audio segments. This pipeline is designed to run on local machines given that it requires access to write to an output device.
+
+## Example
+
+The following shows a simple example using this pipeline.
+
+```python
+from txtai.pipeline import AudioStream
+
+# Create and run pipeline
+audio = AudioStream()
+audio(data)
+```
+
+## Configuration-driven example
+
+Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
+
+### config.yml
+```yaml
+# Create pipeline using lower case class name
+audiostream:
+
+# Run pipeline with workflow
+workflow:
+ audiostream:
+ tasks:
+ - action: audiostream
+```
+
+### Run with Workflows
+
+```python
+from txtai import Application
+
+# Create and run pipeline with workflow
+app = Application("config.yml")
+list(app.workflow("audiostream", ["numpy data"]))
+```
+
+### Run with API
+
+```bash
+CONFIG=config.yml uvicorn "txtai.api:app" &
+
+curl \
+ -X POST "http://localhost:8000/workflow" \
+ -H "Content-Type: application/json" \
+ -d '{"name":"audiostream", "elements":["numpy data"]}'
+```
+
+## Methods
+
+Python documentation for the pipeline.
+
+### ::: txtai.pipeline.AudioStream.__init__
+### ::: txtai.pipeline.AudioStream.__call__
diff --git a/docs/pipeline/index.md b/docs/pipeline/index.md
index 67dbbc12..4b831a96 100644
--- a/docs/pipeline/index.md
+++ b/docs/pipeline/index.md
@@ -12,6 +12,8 @@ Pipelines are run with Python or configuration. Pipelines can be instantiated in
The following is a list of the default pipelines available in txtai. All pipelines use default models when otherwise not specified. See the [model guide](../models) for the current model recommentations.
- Audio
+ - [AudioStream](audio/audiostream)
+ - [Microphone](audio/microphone)
- [TextToSpeech](audio/texttospeech)
- [Transcription](audio/transcription)
- Data Processing
diff --git a/mkdocs.yml b/mkdocs.yml
index e281f49c..8846c420 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -74,6 +74,8 @@ nav:
- Pipeline:
- pipeline/index.md
- Audio:
+ - Audio Stream: pipeline/audio/audiostream.md
+ - Microphone: pipeline/audio/microphone.md
- Text To Speech: pipeline/audio/texttospeech.md
- Transcription: pipeline/audio/transcription.md
- Data:
diff --git a/setup.py b/setup.py
index 5c9994e4..70ea4bff 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,17 @@ extras["graph"] = ["grand-cypher>=0.6.0", "grand-graph>=0.5.0", "networkx>=2.6.3
extras["model"] = ["onnx>=1.11.0", "onnxruntime>=1.11.0"]
-extras["pipeline-audio"] = ["onnx>=1.11.0", "onnxruntime>=1.11.0", "soundfile>=0.10.3.post1", "scipy>=1.4.1", "ttstokenizer>=1.0.0"]
+extras["pipeline-audio"] = [
+ "onnx>=1.11.0",
+ "onnxruntime>=1.11.0",
+ "pyaudio>=0.2.14",
+ "scipy>=1.4.1",
+ "speechrecognition>=3.10.4",
+ "sounddevice>=0.5.0",
+ "soundfile>=0.10.3.post1",
+ "ttstokenizer>=1.0.0",
+ "webrtcvad-wheels>=2.0.14",
+]
extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]
diff --git a/src/python/txtai/pipeline/audio/audiostream.py b/src/python/txtai/pipeline/audio/audiostream.py
new file mode 100644
index 00000000..9feb9135
--- /dev/null
+++ b/src/python/txtai/pipeline/audio/audiostream.py
@@ -0,0 +1,70 @@
+"""
+AudioStream module
+"""
+
+from queue import Queue
+from threading import Thread
+
+# Conditional import
+try:
+ import sounddevice as sd
+
+ SOUNDDEVICE = True
+except ImportError:
+ SOUNDDEVICE = False
+
+from ..base import Pipeline
+
+
+class AudioStream(Pipeline):
+ """
+ Threaded pipeline that streams audio segments to an output audio device. This pipeline is designed
+ to run on local machines given that it requires access to write to an output device.
+ """
+
+ COMPLETE = 1
+
+ def __init__(self, rate=22050):
+ """
+ Creates an AudioStream pipeline.
+
+ Args:
+ rate: sample rate of audio segments
+ """
+
+ if not SOUNDDEVICE:
+ raise ImportError('AudioStream pipeline is not available - install "pipeline" extra to enable')
+
+ # Sampler rate
+ self.rate = rate
+
+ self.queue = Queue()
+ self.thread = Thread(target=self.play)
+ self.thread.start()
+
+ def __call__(self, segment):
+ # Convert single element to list
+ segments = [segment] if not isinstance(segment, list) else segment
+
+ for x in segments:
+ self.queue.put(x)
+
+ # Return single element if single element passed in
+ return segment[0] if not isinstance(segment, list) else segment
+
+ def wait(self):
+ """
+ Waits for all input audio segments to be played.
+ """
+
+ self.thread.join()
+
+ def play(self):
+ """
+ Reads audio segments from queue. This method runs in a separate non-blocking thread.
+ """
+
+ audio = self.queue.get()
+ while not isinstance(audio, int) or audio != AudioStream.COMPLETE:
+ sd.play(audio, self.rate, blocking=True)
+ audio = self.queue.get()
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index f1df867e..6d1ba14e 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -45,7 +45,9 @@ class TestOptional(unittest.TestCase):
"rich",
"sklearn.decomposition",
"sentence_transformers",
+ "sounddevice",
"soundfile",
+ "speech_recognition",
"sqlalchemy",
"sqlite_vec",
"tika",
@@ -178,12 +180,14 @@ class TestOptional(unittest.TestCase):
"""
from txtai.pipeline import (
+ AudioStream,
Caption,
HFOnnx,
HFTrainer,
ImageHash,
LiteLLM,
LlamaCpp,
+ Microphone,
MLOnnx,
Objects,
Segmentation,
@@ -194,6 +198,9 @@ class TestOptional(unittest.TestCase):
Translation,
)
+ with self.assertRaises(ImportError):
+ AudioStream()
+
with self.assertRaises(ImportError):
Caption()
@@ -212,6 +219,9 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
LlamaCpp("TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf")
+ with self.assertRaises(ImportError):
+ Microphone()
+
with self.assertRaises(ImportError):
MLOnnx()
diff --git a/test/python/testpipeline/testaudiostream.py b/test/python/testpipeline/testaudiostream.py
new file mode 100644
index 00000000..4a9529e4
--- /dev/null
+++ b/test/python/testpipeline/testaudiostream.py
@@ -0,0 +1,37 @@
+"""
+AudioStream module tests
+"""
+
+import unittest
+
+from unittest.mock import patch
+
+import soundfile as sf
+
+from txtai.pipeline import AudioStream
+
+# pylint: disable = C0411
+from utils import Utils
+
+
+class TestAudioStream(unittest.TestCase):
+ """
+ AudioStream tests.
+ """
+
+ @patch("sounddevice.play")
+ def testAudioStream(self, play):
+ """
+ Test playing audio
+ """
+
+ play.return_value = True
+
+ # Read audio data
+ raw, samplerate = sf.read(Utils.PATH + "/Make_huge_profits.wav")
+
+ audio = AudioStream(samplerate)
+ self.assertIsNotNone(audio([raw, AudioStream.COMPLETE]))
+
+ # Wait for completion
+ audio.wait()