SOTA Open Source TTS
Greatly reduced cpu usage
Lengyue0687dd1f1a8f
diff --git a/preparing_data/whisper_asr.py b/preparing_data/whisper_asr.pyindex 2216440..c5a0ce4 100644--- a/preparing_data/whisper_asr.py+++ b/preparing_data/whisper_asr.py@@ -1,14 +1,12 @@# This file is used to convert the audio files to text files using the Whisper model.# It's mainly used to generate the training data for the VQ model.-import sysimport torchimport clickimport timefrom transformers import WhisperProcessorfrom speech_lm.models.flash_whisper import FlashWhisperForConditionalGenerationfrom functools import lru_cache-import librosafrom loguru import loggerimport subprocess as spimport os@@ -16,7 +14,8 @@ import torchfrom pathlib import Pathfrom random import Randomfrom datetime import timedelta-import torchaudio+from whisper.audio import log_mel_spectrogram, load_audio, pad_or_trim+import numpy as npRANK_STR = ""@@ -24,7 +23,7 @@ RANK_STR = ""@lru_cache(maxsize=1)def get_whisper_model():model = FlashWhisperForConditionalGeneration.from_pretrained(- "openai/whisper-small"+ "openai/whisper-medium").cuda()model.eval()logger.info(f"{RANK_STR}Loaded model")@@ -34,28 +33,28 @@ def get_whisper_model():@lru_cache(maxsize=1)def get_whisper_processor():- return WhisperProcessor.from_pretrained("openai/whisper-small")+ return WhisperProcessor.from_pretrained("openai/whisper-medium")def transcribe_batch(files: list[str]):- wavs = [librosa.load(file, sr=16000, mono=True)[0] for file in files]+ wavs = [load_audio(file, 16000) for file in files]total_time = sum([len(wav) for wav in wavs]) / 16000+ wavs = [pad_or_trim(wav) for wav in wavs]+ wavs = torch.from_numpy(np.stack(wavs)).float().cuda()+ mels = log_mel_spectrogram(wavs).cuda()model = get_whisper_model()- processor = get_whisper_processor()-- encoded = processor(wavs, sampling_rate=16000, return_tensors="pt")-- input_features = encoded.input_features.cuda()with torch.no_grad():outputs = model.generate(- input_features=input_features,+ input_features=mels,max_length=448,do_sample=False,)+ processor = get_whisper_processor()transcriptions = processor.batch_decode(outputs, skip_special_tokens=True)+return transcriptions, total_time@@ -90,7 +89,7 @@ def main(folder: str, rank: int, world_size: int, num_workers: int):str(num_workers),folder,]- processes.append(+ processes.append(sp.Popen(args,env=env,@@ -133,7 +132,9 @@ def main(folder: str, rank: int, world_size: int, num_workers: int):if (n_batch + 1) % 10 == 0:eta = (- (time.time() - begin_time) / processed_files * (len(files) - processed_files)+ (time.time() - begin_time)+ / processed_files+ * (len(files) - processed_files))logger.info(f"{RANK_STR}Processed {processed_files} files, {total_time / 3600:.2f} hours of audio, ETA: {timedelta(seconds=round(eta))}s"