Skip to content

SOTA Open Source TTS

Add hubert dataset and hubert model

Lengyue311a8b64eae1
diff --git a/speech_lm/datasets/hubert_vq.py b/speech_lm/datasets/hubert_vq.py
new file mode 100644
index 0000000..10ef183
--- /dev/null
+++ b/speech_lm/datasets/hubert_vq.py
@@ -0,0 +1,64 @@
+import librosa
+from torch.utils.data import Dataset
+from pathlib import Path
+import torch
+
+
+class HubertVQDataset(Dataset):
+ def __init__(self, filelist):
+ super().__init__()
+
+ self.files = Path(filelist).read_text().splitlines()
+
+ def __len__(self):
+ return len(self.files)
+
+ def __getitem__(self, idx):
+ wav, _ = librosa.load(self.files[idx], sr=16000, mono=True)
+ wav = torch.from_numpy(wav).float()
+
+ return wav
+
+
+def collate_fn(batch):
+ # -> {"input_values": ..., "attention_mask": ...}
+ max_length = max([len(x) for x in batch])
+
+ input_values = []
+ attention_mask = []
+
+ for x in batch:
+ x_length = len(x)
+ x = torch.nn.functional.pad(x, (0, max_length - x_length))
+ mask = torch.ones_like(x)
+ mask[x_length:] = 0
+
+ input_values.append(x)
+ attention_mask.append(mask)
+
+ input_values = torch.stack(input_values)
+ attention_mask = torch.stack(attention_mask)
+
+ return {"input_values": input_values, "attention_mask": attention_mask}
+
+
+if __name__ == "__main__":
+ from torch.utils.data import DataLoader
+ from transformers import HubertForCTC, Wav2Vec2Processor
+ import soundfile as sf
+
+ dataset = HubertVQDataset("libritts-r.filelist")
+ dataloader = DataLoader(dataset, batch_size=16, shuffle=True, collate_fn=collate_fn)
+ hubert = HubertForCTC.from_pretrained("facebook/hubert-large-ls960-ft")
+ processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft")
+ hubert.eval()
+
+ for batch in dataloader:
+ print(batch)
+ logits = hubert(**batch).logits
+ predicted_ids = torch.argmax(logits, dim=-1)
+ transcription = processor.decode(predicted_ids[0])
+ print(transcription)
+
+ sf.write("test.wav", batch["input_values"][0].numpy(), 16000)
+ break
diff --git a/speech_lm/models/hubert_vq.py b/speech_lm/models/hubert_vq.py
new file mode 100644
index 0000000..a474b73
--- /dev/null
+++ b/speech_lm/models/hubert_vq.py
@@ -0,0 +1,219 @@
+from typing import Optional
+from transformers import HubertModel
+from torch import nn
+import torch
+from encodec.quantization.core_vq import VectorQuantization
+
+
+class HubertVQ(nn.Module):
+ def __init__(
+ self,
+ model_name_or_path: str = "facebook/hubert-large-ls960-ft",
+ vq_layer: int = -4, # the layer to extract the quantized features
+ codebook_size: int = 1024,
+ trainable_layers_before_vq: int = 2,
+ trainable_layers_after_vq: int = 2,
+ ):
+ super().__init__()
+
+ self.hubert = HubertModel.from_pretrained(model_name_or_path)
+ self.vq_layer = (
+ (self.hubert.config.num_hidden_layers + vq_layer)
+ if vq_layer < 0
+ else vq_layer
+ )
+ self.trainable_layers_before_vq = trainable_layers_before_vq
+ self.trainable_layers_after_vq = trainable_layers_after_vq
+
+ assert (
+ self.vq_layer >= trainable_layers_before_vq
+ and self.vq_layer
+ < self.hubert.config.num_hidden_layers - trainable_layers_after_vq
+ ), "vq_layer must be between trainable_layers_before_vq and num_hidden_layers - trainable_layers_after_vq"
+
+ # Freeze both feature extractor & lm head
+ for param in self.hubert.parameters():
+ param.requires_grad = False
+
+ # Unfreeze layers between vq_layer - trainable_layers_before_vq and vq_layer + trainable_layers_after_vq
+ for param in self.hubert.encoder.layers[
+ self.vq_layer
+ - trainable_layers_before_vq : self.vq_layer
+ + trainable_layers_after_vq
+ ].parameters():
+ param.requires_grad = True
+
+ # Quantization
+ self.quantizer = VectorQuantization(
+ codebook_size=codebook_size,
+ dim=self.hubert.config.hidden_size,
+ kmeans_init=False,
+ )
+
+ @torch.no_grad()
+ def _get_attention_mask(
+ self, hidden_states: torch.Tensor, attention_mask: torch.Tensor
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ # compute reduced attention_mask corresponding to feature vectors
+ attention_mask = self.hubert._get_feature_vector_attention_mask(
+ hidden_states.shape[1], attention_mask
+ )
+
+ # make sure padded tokens are not attended to
+ expand_attention_mask = attention_mask.unsqueeze(-1).repeat(
+ 1, 1, hidden_states.shape[2]
+ )
+ hidden_states[~expand_attention_mask] = 0
+
+ # extend attention_mask
+ attention_mask = 1.0 - attention_mask[:, None, None, :].to(
+ dtype=hidden_states.dtype
+ )
+ attention_mask = attention_mask * torch.finfo(hidden_states.dtype).min
+ attention_mask = attention_mask.expand(
+ attention_mask.shape[0],
+ 1,
+ attention_mask.shape[-1],
+ attention_mask.shape[-1],
+ )
+
+ return hidden_states, attention_mask
+
+ def encode(
+ self,
+ input_values: Optional[torch.Tensor],
+ attention_mask: Optional[torch.Tensor] = None,
+ mask_time_indices: Optional[torch.FloatTensor] = None,
+ ) -> torch.Tensor:
+ with torch.no_grad():
+ # Extract features
+ extract_features = self.hubert.feature_extractor(input_values)
+ extract_features = extract_features.transpose(1, 2)
+
+ hidden_states = self.hubert.feature_projection(extract_features)
+ hidden_states = self.hubert._mask_hidden_states(
+ hidden_states, mask_time_indices=mask_time_indices
+ )
+
+ position_embeddings = self.hubert.encoder.pos_conv_embed(hidden_states)
+ hidden_states = hidden_states + position_embeddings
+
+ if attention_mask is not None:
+ # compute reduced attention_mask corresponding to feature vectors
+ hidden_states, attention_mask = self._get_attention_mask(
+ hidden_states, attention_mask
+ )
+
+ # Only do layer norm if do_stable_layer_norm is False
+ if self.hubert.config.do_stable_layer_norm is False:
+ hidden_states = self.hubert.encoder.layer_norm(hidden_states)
+
+ hidden_states = self.hubert.encoder.dropout(hidden_states)
+
+ # Execute transformer
+ for idx, layer_module in enumerate(self.hubert.encoder.layers[: self.vq_layer]):
+ if idx < self.vq_layer - self.trainable_layers_before_vq:
+ with torch.no_grad():
+ hidden_states = layer_module(hidden_states, attention_mask)[0]
+ else:
+ hidden_states = layer_module(hidden_states, attention_mask)[0]
+
+ return hidden_states
+
+ @torch.no_grad()
+ def decode(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ if attention_mask is not None:
+ # compute reduced attention_mask corresponding to feature vectors
+ _, attention_mask = self._get_attention_mask(
+ hidden_states.clone(), attention_mask
+ )
+
+ # Execute transformer
+ for idx, layer_module in enumerate(self.hubert.encoder.layers[self.vq_layer :]):
+ if idx >= self.trainable_layers_after_vq:
+ with torch.no_grad():
+ hidden_states = layer_module(hidden_states, attention_mask)[0]
+ else:
+ hidden_states = layer_module(hidden_states, attention_mask)[0]
+
+ with torch.no_grad():
+ # Only do layer norm if do_stable_layer_norm is False
+ if self.hubert.config.do_stable_layer_norm is False:
+ hidden_states = self.hubert.encoder.last_layer_norm(hidden_states)
+ else:
+ hidden_states = self.hubert.encoder.layer_norm(hidden_states)
+
+ return hidden_states
+
+ def forward(
+ self,
+ input_values: Optional[torch.Tensor],
+ attention_mask: Optional[torch.Tensor] = None,
+ mask_time_indices: Optional[torch.FloatTensor] = None,
+ ):
+ hidden_states = self.encode(
+ input_values,
+ attention_mask=attention_mask,
+ mask_time_indices=mask_time_indices,
+ )
+
+ # Quantize
+ quantize, _, vq_loss = self.quantizer(hidden_states.transpose(1, 2))
+ quantize = quantize.transpose(1, 2)
+
+ # Inject position embeddings
+ with torch.no_grad():
+ position_embeddings = self.hubert.encoder.pos_conv_embed(hidden_states)
+
+ quantize = quantize + position_embeddings
+
+ # Decode
+ hidden_states = self.decode(quantize, attention_mask=attention_mask)
+
+ return hidden_states, vq_loss
+
+
+# class HubertVQ
+if __name__ == "__main__":
+ from transformers import Wav2Vec2Tokenizer
+ from datasets import load_dataset
+
+ processor = Wav2Vec2Tokenizer.from_pretrained("facebook/hubert-large-ls960-ft")
+ model = HubertVQ()
+ model.train()
+ print("Loaded model")
+
+ optim = torch.optim.Adam(model.parameters(), lr=1e-4)
+
+ gt_hubert = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")
+ gt_hubert.train()
+ print("Loaded ground truth model")
+
+ ds = load_dataset(
+ "patrickvonplaten/librispeech_asr_dummy", "clean", split="validation"
+ )
+ print("Loaded dataset")
+
+ input_values = processor(
+ ds[0]["audio"]["array"], return_tensors="pt"
+ ) # Batch size 1
+
+ optim.zero_grad()
+ # hidden_states = model.decode(model.encode(**input_values))
+ hidden_states, vq_loss = model(**input_values)
+ print(hidden_states, vq_loss)
+
+ gt = gt_hubert(**input_values).last_hidden_state
+
+ loss = torch.nn.functional.mse_loss(hidden_states, gt)
+ print(loss)
+
+ total_loss = loss + vq_loss
+ total_loss.backward()
+ optim.step()
+
+ print("Backward pass done")