💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add transcription pipeline, closes #67
davidmezzettie9f162bd477b
diff --git a/setup.py b/setup.pyindex 64cc922b..bebf7608 100644--- a/setup.py+++ b/setup.py@@ -38,6 +38,7 @@ setup("torch>=1.4.0","tqdm>=4.46.0","sentence-transformers>=0.4.0",+ "soundfile>=0.10.3.post1","transformers>=4.4.1","uvicorn>=0.12.1",],diff --git a/src/python/txtai/pipeline/transcription.py b/src/python/txtai/pipeline/transcription.pynew file mode 100644index 00000000..38176648--- /dev/null+++ b/src/python/txtai/pipeline/transcription.py@@ -0,0 +1,105 @@+"""+Transcription module+"""++import soundfile as sf++from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor++from .hfmodel import HFModel+++class Transcription(HFModel):+ """+ Transcribes audio files to text.+ """++ def __init__(self, path="facebook/wav2vec2-base-960h", quantize=False, gpu=True, batch=64):+ """+ Constructs a new transcription pipeline.++ Args:+ path: optional path to model, accepts Hugging Face model hub id or local path,+ uses default model for task if not provided+ quantize: if model should be quantized, defaults to False+ gpu: True/False if GPU should be enabled, also supports a GPU device id+ batch: batch size used to incrementally process content+ langdetect: path to language detection model, uses a default path if not provided+ """++ # Call parent constructor+ super().__init__(path, quantize, gpu, batch)++ # load model and processor+ self.model = Wav2Vec2ForCTC.from_pretrained(self.path)+ self.processor = Wav2Vec2Processor.from_pretrained(self.path)++ # Move model to device+ self.model.to(self.device)++ def __call__(self, files):+ """+ Transcribes audio files to text.++ This method supports files as a string or a list. If the input is a string,+ the return type is string. If text is a list, the return type is a list.++ Args:+ files: text|list++ Returns:+ list of transcribed text+ """++ values = [files] if not isinstance(files, list) else files++ # Parse audio files+ speech = [sf.read(f) for f in values]++ # Get unique list of sampling rates+ unique = set(s[1] for s in speech)++ results = {}+ for sampling in unique:+ # Get inputs for current sampling rate+ inputs = [(x, s[0]) for x, s in enumerate(speech) if s[1] == sampling]++ # Transcribe in batches+ outputs = []+ for chunk in self.batch([s for _, s in inputs], self.batchsize):+ outputs.extend(self.transcribe(chunk, sampling))++ # Store output value+ for y, (x, _) in enumerate(inputs):+ results[x] = outputs[y].capitalize()++ # Return results in same order as input+ results = [results[x] for x in sorted(results)]+ return results[0] if isinstance(files, str) else results++ def transcribe(self, speech, sampling):+ """+ Transcribes audio to text.++ Args:+ speech: list of audio+ sampling: sampling rate++ Returns:+ list of transcribed text+ """++ with self.context():+ # Convert samples to features+ inputs = self.processor(speech, sampling_rate=sampling, padding=True, return_tensors=self.tensortype()).input_values++ # Place inputs on tensor device+ inputs = inputs.to(self.device)++ # Retrieve logits+ logits = self.model(inputs).logits++ # Decode argmax+ ids = self.argmax(logits, dimension=-1)++ return self.processor.batch_decode(ids)