Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add transcription pipeline, closes #67

davidmezzettie9f162bd477b
diff --git a/setup.py b/setup.py
index 64cc922b..bebf7608 100644
--- a/setup.py
+++ b/setup.py
@@ -38,6 +38,7 @@ setup(
"torch>=1.4.0",
"tqdm>=4.46.0",
"sentence-transformers>=0.4.0",
+ "soundfile>=0.10.3.post1",
"transformers>=4.4.1",
"uvicorn>=0.12.1",
],
diff --git a/src/python/txtai/pipeline/transcription.py b/src/python/txtai/pipeline/transcription.py
new file mode 100644
index 00000000..38176648
--- /dev/null
+++ b/src/python/txtai/pipeline/transcription.py
@@ -0,0 +1,105 @@
+"""
+Transcription module
+"""
+
+import soundfile as sf
+
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+
+from .hfmodel import HFModel
+
+
+class Transcription(HFModel):
+ """
+ Transcribes audio files to text.
+ """
+
+ def __init__(self, path="facebook/wav2vec2-base-960h", quantize=False, gpu=True, batch=64):
+ """
+ Constructs a new transcription pipeline.
+
+ Args:
+ path: optional path to model, accepts Hugging Face model hub id or local path,
+ uses default model for task if not provided
+ quantize: if model should be quantized, defaults to False
+ gpu: True/False if GPU should be enabled, also supports a GPU device id
+ batch: batch size used to incrementally process content
+ langdetect: path to language detection model, uses a default path if not provided
+ """
+
+ # Call parent constructor
+ super().__init__(path, quantize, gpu, batch)
+
+ # load model and processor
+ self.model = Wav2Vec2ForCTC.from_pretrained(self.path)
+ self.processor = Wav2Vec2Processor.from_pretrained(self.path)
+
+ # Move model to device
+ self.model.to(self.device)
+
+ def __call__(self, files):
+ """
+ Transcribes audio files to text.
+
+ This method supports files as a string or a list. If the input is a string,
+ the return type is string. If text is a list, the return type is a list.
+
+ Args:
+ files: text|list
+
+ Returns:
+ list of transcribed text
+ """
+
+ values = [files] if not isinstance(files, list) else files
+
+ # Parse audio files
+ speech = [sf.read(f) for f in values]
+
+ # Get unique list of sampling rates
+ unique = set(s[1] for s in speech)
+
+ results = {}
+ for sampling in unique:
+ # Get inputs for current sampling rate
+ inputs = [(x, s[0]) for x, s in enumerate(speech) if s[1] == sampling]
+
+ # Transcribe in batches
+ outputs = []
+ for chunk in self.batch([s for _, s in inputs], self.batchsize):
+ outputs.extend(self.transcribe(chunk, sampling))
+
+ # Store output value
+ for y, (x, _) in enumerate(inputs):
+ results[x] = outputs[y].capitalize()
+
+ # Return results in same order as input
+ results = [results[x] for x in sorted(results)]
+ return results[0] if isinstance(files, str) else results
+
+ def transcribe(self, speech, sampling):
+ """
+ Transcribes audio to text.
+
+ Args:
+ speech: list of audio
+ sampling: sampling rate
+
+ Returns:
+ list of transcribed text
+ """
+
+ with self.context():
+ # Convert samples to features
+ inputs = self.processor(speech, sampling_rate=sampling, padding=True, return_tensors=self.tensortype()).input_values
+
+ # Place inputs on tensor device
+ inputs = inputs.to(self.device)
+
+ # Retrieve logits
+ logits = self.model(inputs).logits
+
+ # Decode argmax
+ ids = self.argmax(logits, dimension=-1)
+
+ return self.processor.batch_decode(ids)