SOTA Open Source TTS
Support pytorch lightning
Lengyuef7f2c03282f4
diff --git a/fish_speech/callbacks/__init__.py b/fish_speech/callbacks/__init__.pynew file mode 100644index 0000000..bbcf3f3--- /dev/null+++ b/fish_speech/callbacks/__init__.py@@ -0,0 +1,3 @@+from .grad_norm import GradNormMonitor++__all__ = ["GradNormMonitor"]diff --git a/fish_speech/callbacks/grad_norm.py b/fish_speech/callbacks/grad_norm.pynew file mode 100644index 0000000..4eb961e--- /dev/null+++ b/fish_speech/callbacks/grad_norm.py@@ -0,0 +1,86 @@+from typing import Union++import lightning.pytorch as pl+import torch+from lightning import LightningModule, Trainer+from lightning.pytorch.callbacks import Callback+from torch import Tensor+from torch.utils._foreach_utils import (+ _group_tensors_by_device_and_dtype,+ _has_foreach_support,+)++[email protected]_grad()+def grad_norm(+ parameters: Union[Tensor, list[Tensor]],+ norm_type: float = 2.0,+) -> float:+ """+ Returns the norm of the gradients of the given parameters.++ Args:+ parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a+ single Tensor that will have gradients normalized+ norm_type (float): type of the used p-norm.++ Returns:+ Total norm of the parameter gradients (viewed as a single vector).+ """ # noqa: E501++ if isinstance(parameters, Tensor):+ parameters = [parameters]++ grads = [p.grad for p in parameters if p.grad is not None]+ first_device = grads[0].device+ grouped_grads: dict[+ tuple[torch.device, torch.dtype], list[list[Tensor]]+ ] = _group_tensors_by_device_and_dtype(+ [[g.detach() for g in grads]]+ ) # type: ignore[assignment]++ norms = []+ for (device, _), ([grads], _) in grouped_grads.items():+ if _has_foreach_support(grads, device=device):+ norms.extend(torch._foreach_norm(grads, norm_type))+ else:+ norms.extend([torch.norm(g, norm_type) for g in grads])++ return torch.norm(torch.stack([norm.to(first_device) for norm in norms]), norm_type)+++class GradNormMonitor(Callback):+ """+ Callback that computes the gradient norm of the model parameters.+ """++ def __init__(self, norm_type: float = 2.0, logging_interval: str = "step") -> None:+ """+ Args:+ norm_type (float): type of the used p-norm.+ logging_interval (str): "step" or "epoch".+ """+ super().__init__()+ self.norm_type = norm_type+ self.logging_interval = logging_interval++ def on_after_backward(self, trainer: Trainer, model: LightningModule) -> None:+ """+ Computes the gradient norm of the model parameters and logs it to the logger.++ Args:+ trainer (Trainer): The trainer object+ model (LightningModule): The current lightningModule+ """++ grad_norm_val = grad_norm(model.parameters(), self.norm_type)++ model_name = model.__class__.__name__.lower()++ on_step = self.logging_interval == "step"+ model.log(+ f"train/{model_name}/grad_norm",+ grad_norm_val,+ on_step=on_step,+ on_epoch=not on_step,+ )diff --git a/fish_speech/configs/base.yaml b/fish_speech/configs/base.yamlnew file mode 100644index 0000000..ead0f86--- /dev/null+++ b/fish_speech/configs/base.yaml@@ -0,0 +1,74 @@+# Base configuration for training a model+paths:+ run_dir: results/${project}+ ckpt_dir: ${paths.run_dir}/checkpoints++hydra:+ run:+ dir: ${paths.run_dir}++# Lightning Trainer+trainer:+ _target_: lightning.pytorch.trainer.Trainer++ default_root_dir: ${paths.run_dir}+ accelerator: gpu+ num_nodes: 1+ devices: 8+ strategy:+ _target_: lightning.pytorch.strategies.DDPStrategy+ static_graph: true+ precision: bf16-mixed++ # disable validation by epoch end+ check_val_every_n_epoch: null+ val_check_interval: 5000+ max_steps: 100_000++ # Use torch.backends.cudnn.benchmark to speed up training+ benchmark: true++# Callbacks+callbacks:+ model_checkpoint:+ _target_: lightning.pytorch.callbacks.ModelCheckpoint+ dirpath: ${paths.ckpt_dir}+ filename: "step_{step:09d}"+ save_last: true # additionally always save an exact copy of the last checkpoint to a file last.ckpt+ save_top_k: 5 # save 5 latest checkpoints+ monitor: step # use step to monitor checkpoints+ mode: max # save the latest checkpoint with the highest global_step+ every_n_epochs: null # don't save checkpoints by epoch end+ every_n_train_steps: 5000 # save checkpoints every 5000 steps+ auto_insert_metric_name: false++ model_summary:+ _target_: lightning.pytorch.callbacks.RichModelSummary+ max_depth: 2 # the maximum depth of layer nesting that the summary will include++ rich_progress_bar:+ _target_: lightning.pytorch.callbacks.RichProgressBar++ learning_rate_monitor:+ _target_: lightning.pytorch.callbacks.LearningRateMonitor+ logging_interval: step+ log_momentum: false++ grad_norm_monitor:+ _target_: fish_speech.callbacks.GradNormMonitor+ norm_type: 2+ logging_interval: step++# Logger+logger:+ tensorboard:+ _target_: lightning.pytorch.loggers.tensorboard.TensorBoardLogger+ save_dir: "${paths.run_dir}/tensorboard/"+ name: null+ log_graph: false+ default_hp_metric: true+ prefix: ""++# Loop+train: true+test: falsediff --git a/fish_speech/configs/llama_finetune.yaml b/fish_speech/configs/llama_finetune.yamlindex 4c0ee2c..f0f08e2 100644--- a/fish_speech/configs/llama_finetune.yaml+++ b/fish_speech/configs/llama_finetune.yaml@@ -1,116 +1,73 @@-paths:- run_dir: results/finetune- checkpoint_dir: ${paths.run_dir}/checkpoints+defaults:+ - base+ - _self_-hydra:- run:- dir: ${paths.run_dir}+project: llama_finetune+# Lightning Trainertrainer:- _target_: lightning.fabric.Fabric- accelerator: gpu- strategy:- _target_: lightning.fabric.strategies.DDPStrategy- static_graph: true- num_nodes: 1- devices: 8- precision: bf16-mixed- loggers:- _target_: pytorch_lightning.loggers.TensorBoardLogger- save_dir: ${paths.run_dir}- name: tensorboard- version: null--model:- _target_: transformers.AutoModelForCausalLM.from_pretrained- pretrained_model_name_or_path: fishaudio/speech-lm-300m- revision: text-pretrain-10k+ accumulate_grad_batches: 2+ gradient_clip_val: 1.0+ gradient_clip_algorithm: 'norm'+# Dataset Configurationtokenizer:_target_: transformers.AutoTokenizer.from_pretrainedpretrained_model_name_or_path: fishaudio/speech-lm-300mrevision: text-pretrain-10k-# This is a 200 billion seen token schedule-schedule:- max_length: 1024- batch_size: 16 # 128 * 4 = 512- micro_batch_size: 8- max_steps: 100000- save_interval: 5000- log_interval: 10- gradient_accumulation_steps: "${eval: ${schedule.batch_size} // ${schedule.micro_batch_size}}"- clip_grad_norm: 1.0-+# Dataset Configurationtrain_dataset:- _target_: fish_speech.datasets.cultura_x.InterleaveDataset+ _target_: fish_speech.datasets.text.InterleaveDatasetdatasets:- - _target_: fish_speech.datasets.cultura_x.CulturaXDataset- lang: 'en'- - _target_: fish_speech.datasets.cultura_x.CulturaXDataset- lang: 'zh'- - _target_: fish_speech.datasets.cultura_x.CulturaXDataset- lang: 'ja'- - _target_: fish_speech.datasets.cultura_x.CulturaXDataset+ - _target_: fish_speech.datasets.text.TextDataset+ prefix: 'en/'+ - _target_: fish_speech.datasets.text.TextDataset+ prefix: 'zh/'+ - _target_: fish_speech.datasets.text.TextDataset+ prefix: 'ja/'+ - _target_: fish_speech.datasets.text.TextDatasetrepo: fishaudio/wenet-vq- files:- - data/train-00000-of-00018-b5a82c6054c6acca.parquet- - data/train-00001-of-00018-82467b3e0669c2be.parquet- - data/train-00002-of-00018-d50ed8c218a1f183.parquet- - data/train-00003-of-00018-15d666053eade100.parquet- - data/train-00004-of-00018-01868cb8408e012b.parquet- - data/train-00005-of-00018-e766a0b54b1fd08b.parquet- - data/train-00006-of-00018-c79fad54ea8a0b8d.parquet- - data/train-00007-of-00018-e4155011a7081a1d.parquet- - data/train-00008-of-00018-8ba319f5af359d15.parquet- - data/train-00009-of-00018-9c9e984a6565b2c3.parquet- - data/train-00010-of-00018-7af80a80e5aa1e54.parquet- - data/train-00011-of-00018-2ab91221787a84a3.parquet- - data/train-00012-of-00018-4d477812eea5d298.parquet- - data/train-00013-of-00018-faf87b68b1ab4a15.parquet- - data/train-00014-of-00018-7f6bbd9bcb4cbb55.parquet- - data/train-00015-of-00018-d630fe4a488b9f51.parquet- - data/train-00016-of-00018-969a4d5dc04d2764.parquet- - data/train-00017-of-00018-bbfd09175809d1fe.parquet+ prefix: 'data/train'probabilities: [0.2, 0.2, 0.2, 0.4]seed: 42-train_dataloader:- _target_: torch.utils.data.DataLoader- dataset: ${train_dataset}- batch_size: ${schedule.micro_batch_size}- num_workers: 8- collate_fn:- _target_: fish_speech.datasets.cultura_x.CulutreXCollator- tokenizer: ${tokenizer}- max_length: ${schedule.max_length}+val_dataset:+ _target_: fish_speech.datasets.text.TextDataset+ repo: fishaudio/wenet-vq+ prefix: 'data/test'-valid_dataloader:- _target_: torch.utils.data.DataLoader- dataset:- _target_: fish_speech.datasets.cultura_x.CulturaXDataset- repo: fishaudio/wenet-vq- files:- - data/test-00000-of-00001-685250c116f5d321.parquet- batch_size: ${schedule.micro_batch_size}+data:+ _target_: fish_speech.datasets.text.TextDataModule+ train_dataset: ${train_dataset}+ val_dataset: ${val_dataset}num_workers: 1- collate_fn:- _target_: fish_speech.datasets.cultura_x.CulutreXCollator- tokenizer: ${tokenizer}- max_length: ${schedule.max_length}+ batch_size: 8+ tokenizer: ${tokenizer}-optimizer:- _target_: torch.optim.AdamW- lr: 1e-4- weight_decay: 0.1- betas: [0.9, 0.95]- eps: 1e-5+# Model Configuration+model:+ _target_: fish_speech.models.text2semantic.TextToSemantic++ model:+ _target_: transformers.AutoModelForCausalLM.from_pretrained+ pretrained_model_name_or_path: fishaudio/speech-lm-300m+ revision: text-pretrain-10k++ optimizer:+ _target_: torch.optim.AdamW+ _partial_: true+ lr: 1e-4+ weight_decay: 0.1+ betas: [0.9, 0.95]+ eps: 1e-5-scheduler:- _target_: torch.optim.lr_scheduler.LambdaLR- lr_lambda:- _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda+ lr_scheduler:+ _target_: torch.optim.lr_scheduler.LambdaLR_partial_: true- num_warmup_steps: 2000- num_training_steps: ${schedule.max_steps}- final_lr_ratio: 0.1+ lr_lambda:+ _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda+ _partial_: true+ num_warmup_steps: 2000+ num_training_steps: ${trainer.max_steps}+ final_lr_ratio: 0.1diff --git a/fish_speech/datasets/cultura_x.py b/fish_speech/datasets/text.pysimilarity index 57%rename from fish_speech/datasets/cultura_x.pyrename to fish_speech/datasets/text.pyindex 093688b..5046946 100644--- a/fish_speech/datasets/cultura_x.py+++ b/fish_speech/datasets/text.py@@ -1,51 +1,55 @@import randomfrom dataclasses import dataclass-from logging import getLogger+from itertools import chainfrom random import Random-from typing import Optional+from typing import Optional, Unionimport numpy as np-import pandas as pdimport pyarrow.parquet as pqfrom datasets.download.streaming_download_manager import xopen+from huggingface_hub import HfApi+from lightning import LightningDataModulefrom torch.distributed import get_rank, get_world_size, is_initialized-from torch.utils.data import IterableDataset, get_worker_info+from torch.utils.data import DataLoader, IterableDataset, get_worker_infofrom transformers import AutoTokenizer+from fish_speech.utils import RankedLoggerfrom fish_speech.utils.braceexpand import braceexpand-SUBSETS = {- "en": "en_part_{00000..03071}",- "zh": "zh_part_{00000..00319}",- "ja": "ja_part_{00000..00159}",-}+log = RankedLogger(__name__, rank_zero_only=True)-log = getLogger(__name__)--class CulturaXDataset(IterableDataset):+class TextDataset(IterableDataset):def __init__(self,- lang: Optional[str] = None,+ files: Optional[Union[list[str], str]] = None,+ prefix: Optional[str] = None,seed: int = 42,parquet_batch_size: int = 10000,repo: str = "uonlp/CulturaX",- files: Optional[list[str]] = None,):super().__init__()- self.lang = langself.seed = seedself.parquet_batch_size = parquet_batch_sizeself.repo = repo- if self.lang is not None:- files = sorted(list(braceexpand(f"{lang}/{SUBSETS[lang]}.parquet")))+ if files is None and prefix is None:+ raise ValueError("Either files or prefix must be specified")++ if prefix is not None:+ files = HfApi().list_repo_files(repo, repo_type="dataset")+ files = [f for f in files if f.startswith(prefix)]+ log.info(f"Found {len(files)} files in {repo} with prefix {prefix}")else:- files = list(files)+ if isinstance(files, str):+ files = [files]++ files = list(chain.from_iterable(map(braceexpand, files)))+ log.info(f"Expanded {len(files)} files in {repo}")# Get sharded files- self.files = files+ self.files = sorted(files)Random(seed).shuffle(self.files)def get_data_splits(self, files):@@ -100,7 +104,7 @@ class CulturaXDataset(IterableDataset):@dataclass-class CulutreXCollator:+class TextDataCollator:tokenizer: AutoTokenizermax_length: int = 512@@ -154,16 +158,68 @@ class InterleaveDataset(IterableDataset):yield next(dataset_iterators[dataset_idx])-if __name__ == "__main__":- from torch.utils.data import DataLoader+class TextDataModule(LightningDataModule):+ def __init__(+ self,+ train_dataset: Union[TextDataset, InterleaveDataset],+ val_dataset: Optional[Union[TextDataset, InterleaveDataset]] = None,+ batch_size: int = 32,+ tokenizer: AutoTokenizer = None,+ max_length: int = 1024,+ num_workers: int = 4,+ ):+ super().__init__()++ self.train_dataset = train_dataset+ self.val_dataset = val_dataset+ self.batch_size = batch_size+ self.tokenizer = tokenizer+ self.max_length = max_length+ self.num_workers = num_workers++ def train_dataloader(self):+ return DataLoader(+ self.train_dataset,+ batch_size=self.batch_size,+ collate_fn=TextDataCollator(self.tokenizer, self.max_length),+ num_workers=self.num_workers,+ )- from fish_speech.datasets.wenet_vq import WenetVQDataset+ def val_dataloader(self):+ if self.val_dataset is None:+ return None++ return DataLoader(+ self.val_dataset,+ batch_size=self.batch_size,+ collate_fn=TextDataCollator(self.tokenizer, self.max_length),+ num_workers=self.num_workers,+ )- dataset_en = CulturaXDataset("en")- dataset_ja = CulturaXDataset("ja")- dataset_wenet = WenetVQDataset()- dataset = InterleaveDataset([dataset_en, dataset_wenet], [0.5, 0.5])- collator = CulutreXCollator(AutoTokenizer.from_pretrained("gpt2"))- for batch in DataLoader(dataset, batch_size=4, collate_fn=collator, num_workers=4):+if __name__ == "__main__":+ dm = TextDataModule(+ InterleaveDataset(+ datasets=[+ TextDataset(+ prefix="en/en_part_",+ ),+ TextDataset(+ prefix="zh/zh_part_",+ ),+ TextDataset(+ prefix="ja/ja_part_",+ ),+ ],+ probabilities=[0.8, 0.1, 0.1],+ ),+ TextDataset(+ files="ja/ja_part_{00000..00159}",+ ),+ batch_size=2,+ tokenizer=AutoTokenizer.from_pretrained("bert-base-multilingual-cased"),+ )++ for batch in dm.train_dataloader():print(batch)+ breakdiff --git a/fish_speech/models/text2semantic/__init__.py b/fish_speech/models/text2semantic/__init__.pynew file mode 100644index 0000000..cceba5d--- /dev/null+++ b/fish_speech/models/text2semantic/__init__.py@@ -0,0 +1,3 @@+from .lit_module import TextToSemantic++__all__ = ["TextToSemantic"]diff --git a/fish_speech/models/text2semantic/lit_module.py b/fish_speech/models/text2semantic/lit_module.pynew file mode 100644index 0000000..38c09cb--- /dev/null+++ b/fish_speech/models/text2semantic/lit_module.py@@ -0,0 +1,64 @@+from typing import Any++import lightning as L+from lightning.pytorch.utilities.types import OptimizerLRScheduler+from transformers import LlamaForCausalLM+++class TextToSemantic(L.LightningModule):+ def __init__(self, model: LlamaForCausalLM, optimizer: Any, lr_scheduler: Any):+ super().__init__()++ self.model = model+ self.optimizer_builder = optimizer+ self.lr_scheduler_builder = lr_scheduler++ def forward(self, x):+ return self.model(x)++ def configure_optimizers(self) -> OptimizerLRScheduler:+ optimizer = self.optimizer_builder(self.parameters())+ lr_scheduler = self.lr_scheduler_builder(optimizer)++ return {+ "optimizer": optimizer,+ "lr_scheduler": {+ "scheduler": lr_scheduler,+ "interval": "step",+ },+ }++ def _step(self, batch, batch_idx, stage: str):+ result = self.model(**batch)+ loss = result.loss+ logits = result.logits++ self.log(+ f"{stage}/loss",+ loss,+ on_step=True,+ on_epoch=False,+ prog_bar=True,+ logger=True,+ )++ # Top-5 accuracy+ _, indices = logits.topk(5, dim=-1)+ correct = indices.eq(batch["labels"].unsqueeze(-1)).sum()+ accuracy = correct / batch["labels"].numel()+ self.log(+ f"{stage}/accuracy",+ accuracy,+ on_step=True,+ on_epoch=False,+ prog_bar=True,+ logger=True,+ )++ return loss++ def training_step(self, batch, batch_idx):+ return self._step(batch, batch_idx, "train")++ def validation_step(self, batch, batch_idx):+ return self._step(batch, batch_idx, "val")diff --git a/fish_speech/train copy.py b/fish_speech/train copy.pydeleted file mode 100644index b50782e..0000000--- a/fish_speech/train copy.py+++ /dev/null@@ -1,292 +0,0 @@-import time-from collections import defaultdict-from datetime import timedelta-from pathlib import Path-from typing import Optional--import hydra-import torch-from lightning.fabric import Fabric-from natsort import natsorted-from omegaconf import DictConfig, OmegaConf-from tqdm import tqdm-from transformers import LlamaForCausalLM-from transformers.utils import is_flash_attn_available--from fish_speech.logger import RankedLogger--# Allow TF32 on Ampere GPUs-torch.set_float32_matmul_precision("high")-torch.backends.cudnn.allow_tf32 = True--# register eval resolver-OmegaConf.register_new_resolver("eval", eval)--log = RankedLogger(__name__, rank_zero_only=True)---def valid(- model: LlamaForCausalLM,- valid_dataloader: Optional[torch.utils.data.DataLoader],- global_step: int,- fabric: Fabric,- cfg: DictConfig,-):- model.eval()- log.info(f"Evaluating at step {global_step}")-- accumulate_infos = None-- for idx, batch in enumerate(tqdm(valid_dataloader, desc="Evaluating")):- outputs = model(**batch)- loss = outputs.loss- metrics = getattr(outputs, "metrics", {})- log_info = {- "valid/loss": float(loss),- **{f"valid/{k}": float(v) for k, v in metrics.items()},- }-- fabric.log_dict(- log_info,- step=global_step + idx,- )-- # Update log info- if accumulate_infos is None:- accumulate_infos = log_info- else:- assert set(accumulate_infos.keys()) == set(- log_info.keys()- ), "Log keys changed during evaluation"- for k in accumulate_infos.keys():- accumulate_infos[k] += log_info[k]-- if idx == getattr(cfg.schedule, "eval_max_batches", None):- break-- # Log average- items = []- for k in accumulate_infos.keys():- items.append(f"{k}: {accumulate_infos[k] / (idx + 1):.4f}")- log.info(f"Average: {' | '.join(items)}")---def train(- model: LlamaForCausalLM,- optimizer: torch.optim.Optimizer,- scheduler: torch.optim.lr_scheduler._LRScheduler,- train_dataloader: torch.utils.data.DataLoader,- valid_dataloader: Optional[torch.utils.data.DataLoader],- global_step: int,- fabric: Fabric,- cfg: DictConfig,-):- accumulate_steps = 0- optimizer.zero_grad()-- # Start time is ~model forward time + data loading time- start_time = time.time()- trackers = defaultdict(list)-- while global_step < cfg.schedule.max_steps:- last_batch_time = time.time()- for batch in train_dataloader:- # Measure time used by data loading- trackers["data_time"].append(time.time() - last_batch_time)-- # Measure time used by model forward- model_begin_time = time.time()- model.train()-- # Accumulate gradients- gradient_accumulation_steps = cfg.schedule.gradient_accumulation_steps- is_accumulating = accumulate_steps % gradient_accumulation_steps != 0- accumulate_steps += 1-- # Train one step- with fabric.no_backward_sync(model, enabled=is_accumulating):- outputs = model(**batch)- loss = outputs.loss- metrics = getattr(outputs, "metrics", {})-- # Need to divide loss by accumulation steps- fabric.backward(loss / gradient_accumulation_steps)-- # Update trackers- trackers["loss"].append(float(loss))- trackers["lr"].append(float(optimizer.param_groups[0]["lr"]))- for k, v in metrics.items():- trackers[f"metrics/{k}"].append(float(v))-- trackers["model_time"].append(time.time() - model_begin_time)-- if is_accumulating:- last_batch_time = time.time()- continue-- # Check all trackers has the same length- assert (- len(set(len(v) for k, v in trackers.items() if k != "grad_norm")) == 1- ), "Trackers has ambiguous length"-- # Perform gradient clipping- grad_norm = fabric.clip_gradients(- model,- optimizer,- max_norm=cfg.schedule.clip_grad_norm,- norm_type=2.0,- error_if_nonfinite=True,- )-- if torch.isnan(grad_norm) or torch.isinf(grad_norm):- log.warning(f"Gradient norm is {grad_norm}, skipping update")- optimizer.zero_grad()-- # We can't average gradients across multiple steps- trackers["grad_norm"].append(float(grad_norm))-- # Update- optimizer.step()- optimizer.zero_grad()- scheduler.step()-- fabric.log_dict(- {- f"train/{k}": sum(v[-gradient_accumulation_steps:])- / len(v[-gradient_accumulation_steps:])- for k, v in trackers.items()- },- step=global_step,- )-- # accumulate_steps = 0- global_step += 1-- if global_step % cfg.schedule.log_interval == 0:- step_time = (time.time() - start_time) / cfg.schedule.log_interval- eta = step_time * (cfg.schedule.max_steps - global_step)- additional_info = [- f"{k}: {sum(v[-cfg.schedule.log_interval:]) / len(v[-cfg.schedule.log_interval:]):.4f}"- for k, v in trackers.items()- if k != "lr" # lr use .2e format- ]-- log.info(- f"[{global_step}/{cfg.schedule.max_steps}] "- + f"step_time: {step_time:.2f}s "- + f"ETA: {timedelta(seconds=round(eta))}s "- f"lr: {optimizer.param_groups[0]['lr']:.2e} "- + " ".join(additional_info)- )-- # Reset trackers- trackers = defaultdict(list)-- start_time = time.time()-- if global_step % cfg.schedule.save_interval == 0:- fabric.save(- Path(cfg.paths.checkpoint_dir) / f"step_{global_step}.ckpt",- {- "model": model,- "optimizer": optimizer,- "scheduler": scheduler.state_dict(),- "global_step": global_step,- },- )-- if (- getattr(cfg.schedule, "eval_interval", None) is not None- and global_step % cfg.schedule.eval_interval == 0- and valid_dataloader is not None- ):- valid(model, valid_dataloader, global_step, fabric, cfg)-- if global_step >= cfg.schedule.max_steps:- break-- last_batch_time = time.time()--- version_base="1.3", config_path="./configs", config_name="llama_pretrain.yaml"-)-def main(cfg: DictConfig):- log.info(f"Config: \n{OmegaConf.to_yaml(cfg)}")-- if is_flash_attn_available() is False:- log.warning("Flash attention is not available, using default attention")-- fabric: Fabric = hydra.utils.instantiate(cfg.trainer)- fabric.launch()- log.info(f"Fabric: {fabric}")-- model = hydra.utils.instantiate(cfg.model)- log.info(f"Model: {repr(model)}")-- trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)- freeze_params = sum(p.numel() for p in model.parameters() if not p.requires_grad)- log.info(f"Trainable parameters: {trainable_params/1e6:.2f}M")- log.info(f"Freeze parameters: {freeze_params/1e6:.2f}M")-- optimizer = hydra.utils.instantiate(cfg.optimizer, params=model.parameters())- scheduler = hydra.utils.instantiate(cfg.scheduler, optimizer=optimizer)- log.info(f"Optimizer: {optimizer}")- log.info(f"Scheduler: {scheduler}")-- log.info(f"Setup fabric model & dataset")- model = fabric.setup_module(model)- optimizer = fabric.setup_optimizers(optimizer)-- # Build state- global_step = 0-- # Restore training from checkpoint- checkpoint_dir = Path(cfg.paths.checkpoint_dir)- checkpoint_dir.mkdir(parents=True, exist_ok=True)-- # Alphabetically sort checkpoints- checkpoints = natsorted(checkpoint_dir.glob("*.ckpt"))- if len(checkpoints) > 0:- checkpoint_path = checkpoints[-1]-- log.info(f"Restoring checkpoint from {checkpoint_path}")- remainder = fabric.load(- checkpoint_path,- {- "model": model,- "optimizer": optimizer,- "scheduler": scheduler,- },- )- global_step = remainder["global_step"]- log.info(f"Restored global step: {global_step}")-- train_dataloader = hydra.utils.instantiate(cfg.train_dataloader)- log.info(f"Train Dataloader: {train_dataloader}")-- valid_dataloader = None- if getattr(cfg, "valid_dataloader", None) is not None:- valid_dataloader = hydra.utils.instantiate(cfg.valid_dataloader)- log.info(f"Valid Dataloader: {valid_dataloader}")-- train_dataloader = fabric.setup_dataloaders(train_dataloader)- if valid_dataloader is not None:- valid_dataloader = fabric.setup_dataloaders(valid_dataloader)-- log.info(f"Begin training")-- train(- model=model,- optimizer=optimizer,- scheduler=scheduler,- train_dataloader=train_dataloader,- valid_dataloader=valid_dataloader,- global_step=global_step,- fabric=fabric,- cfg=cfg,- )---if __name__ == "__main__":- main()diff --git a/fish_speech/train.py b/fish_speech/train.pyindex b50782e..05967d8 100644--- a/fish_speech/train.py+++ b/fish_speech/train.py@@ -1,19 +1,13 @@-import time-from collections import defaultdict-from datetime import timedelta-from pathlib import Pathfrom typing import Optionalimport hydra+import lightning as Limport torch-from lightning.fabric import Fabric-from natsort import natsorted+from lightning import Callback, LightningDataModule, LightningModule, Trainer+from lightning.pytorch.loggers import Loggerfrom omegaconf import DictConfig, OmegaConf-from tqdm import tqdm-from transformers import LlamaForCausalLM-from transformers.utils import is_flash_attn_available-from fish_speech.logger import RankedLogger+import fish_speech.utils as utils# Allow TF32 on Ampere GPUstorch.set_float32_matmul_precision("high")@@ -22,270 +16,109 @@ torch.backends.cudnn.allow_tf32 = True# register eval resolverOmegaConf.register_new_resolver("eval", eval)-log = RankedLogger(__name__, rank_zero_only=True)---def valid(- model: LlamaForCausalLM,- valid_dataloader: Optional[torch.utils.data.DataLoader],- global_step: int,- fabric: Fabric,- cfg: DictConfig,-):- model.eval()- log.info(f"Evaluating at step {global_step}")-- accumulate_infos = None-- for idx, batch in enumerate(tqdm(valid_dataloader, desc="Evaluating")):- outputs = model(**batch)- loss = outputs.loss- metrics = getattr(outputs, "metrics", {})- log_info = {- "valid/loss": float(loss),- **{f"valid/{k}": float(v) for k, v in metrics.items()},- }-- fabric.log_dict(- log_info,- step=global_step + idx,- )-- # Update log info- if accumulate_infos is None:- accumulate_infos = log_info- else:- assert set(accumulate_infos.keys()) == set(- log_info.keys()- ), "Log keys changed during evaluation"- for k in accumulate_infos.keys():- accumulate_infos[k] += log_info[k]-- if idx == getattr(cfg.schedule, "eval_max_batches", None):- break-- # Log average- items = []- for k in accumulate_infos.keys():- items.append(f"{k}: {accumulate_infos[k] / (idx + 1):.4f}")- log.info(f"Average: {' | '.join(items)}")---def train(- model: LlamaForCausalLM,- optimizer: torch.optim.Optimizer,- scheduler: torch.optim.lr_scheduler._LRScheduler,- train_dataloader: torch.utils.data.DataLoader,- valid_dataloader: Optional[torch.utils.data.DataLoader],- global_step: int,- fabric: Fabric,- cfg: DictConfig,-):- accumulate_steps = 0- optimizer.zero_grad()-- # Start time is ~model forward time + data loading time- start_time = time.time()- trackers = defaultdict(list)-- while global_step < cfg.schedule.max_steps:- last_batch_time = time.time()- for batch in train_dataloader:- # Measure time used by data loading- trackers["data_time"].append(time.time() - last_batch_time)-- # Measure time used by model forward- model_begin_time = time.time()- model.train()-- # Accumulate gradients- gradient_accumulation_steps = cfg.schedule.gradient_accumulation_steps- is_accumulating = accumulate_steps % gradient_accumulation_steps != 0- accumulate_steps += 1-- # Train one step- with fabric.no_backward_sync(model, enabled=is_accumulating):- outputs = model(**batch)- loss = outputs.loss- metrics = getattr(outputs, "metrics", {})-- # Need to divide loss by accumulation steps- fabric.backward(loss / gradient_accumulation_steps)-- # Update trackers- trackers["loss"].append(float(loss))- trackers["lr"].append(float(optimizer.param_groups[0]["lr"]))- for k, v in metrics.items():- trackers[f"metrics/{k}"].append(float(v))-- trackers["model_time"].append(time.time() - model_begin_time)-- if is_accumulating:- last_batch_time = time.time()- continue-- # Check all trackers has the same length- assert (- len(set(len(v) for k, v in trackers.items() if k != "grad_norm")) == 1- ), "Trackers has ambiguous length"-- # Perform gradient clipping- grad_norm = fabric.clip_gradients(- model,- optimizer,- max_norm=cfg.schedule.clip_grad_norm,- norm_type=2.0,- error_if_nonfinite=True,- )+log = utils.RankedLogger(__name__, rank_zero_only=True)++[email protected]_wrapper+def train(cfg: DictConfig) -> tuple[dict, dict]:+ """Trains the model. Can additionally evaluate on a testset, using best weights obtained during+ training.+ This method is wrapped in optional @task_wrapper decorator, that controls the behavior during+ failure. Useful for multiruns, saving info about the crash, etc.+ Args:+ cfg (DictConfig): Configuration composed by Hydra.+ Returns:+ Tuple[dict, dict]: Dict with metrics and dict with all instantiated objects.+ """ # noqa: E501++ # set seed for random number generators in pytorch, numpy and python.random+ if cfg.get("seed"):+ L.seed_everything(cfg.seed, workers=True)++ if cfg.get("deterministic"):+ torch.use_deterministic_algorithms(True)++ log.info(f"Instantiating datamodule <{cfg.data._target_}>")+ datamodule: LightningDataModule = hydra.utils.instantiate(cfg.data)++ log.info(f"Instantiating model <{cfg.model._target_}>")+ model: LightningModule = hydra.utils.instantiate(cfg.model)++ log.info("Instantiating callbacks...")+ callbacks: list[Callback] = utils.instantiate_callbacks(cfg.get("callbacks"))++ log.info("Instantiating loggers...")+ logger: list[Logger] = utils.instantiate_loggers(cfg.get("logger"))++ log.info(f"Instantiating trainer <{cfg.trainer._target_}>")+ trainer: Trainer = hydra.utils.instantiate(+ cfg.trainer, callbacks=callbacks, logger=logger+ )++ object_dict = {+ "cfg": cfg,+ "datamodule": datamodule,+ "model": model,+ "callbacks": callbacks,+ "logger": logger,+ "trainer": trainer,+ }++ if logger:+ log.info("Logging hyperparameters!")+ utils.log_hyperparameters(object_dict)- if torch.isnan(grad_norm) or torch.isinf(grad_norm):- log.warning(f"Gradient norm is {grad_norm}, skipping update")- optimizer.zero_grad()-- # We can't average gradients across multiple steps- trackers["grad_norm"].append(float(grad_norm))-- # Update- optimizer.step()- optimizer.zero_grad()- scheduler.step()-- fabric.log_dict(- {- f"train/{k}": sum(v[-gradient_accumulation_steps:])- / len(v[-gradient_accumulation_steps:])- for k, v in trackers.items()- },- step=global_step,+ if cfg.get("compile"):+ log.info("Compiling model!")+ model = torch.compile(model)++ if cfg.get("train"):+ log.info("Starting training!")++ ckpt_path = cfg.get("ckpt_path")++ if ckpt_path is None:+ ckpt_path = utils.get_latest_checkpoint(cfg.paths.ckpt_dir)++ if ckpt_path is not None:+ log.info(f"Resuming from checkpoint: {ckpt_path}")++ if cfg.get("resume_weights_only"):+ log.info("Resuming weights only!")+ ckpt = torch.load(ckpt_path, map_location=model.device)+ model.load_state_dict(+ ckpt["state_dict"] if "state_dict" in ckpt else ckpt, strict=False)+ ckpt_path = None++ trainer.fit(model=model, datamodule=datamodule, ckpt_path=ckpt_path)- # accumulate_steps = 0- global_step += 1-- if global_step % cfg.schedule.log_interval == 0:- step_time = (time.time() - start_time) / cfg.schedule.log_interval- eta = step_time * (cfg.schedule.max_steps - global_step)- additional_info = [- f"{k}: {sum(v[-cfg.schedule.log_interval:]) / len(v[-cfg.schedule.log_interval:]):.4f}"- for k, v in trackers.items()- if k != "lr" # lr use .2e format- ]-- log.info(- f"[{global_step}/{cfg.schedule.max_steps}] "- + f"step_time: {step_time:.2f}s "- + f"ETA: {timedelta(seconds=round(eta))}s "- f"lr: {optimizer.param_groups[0]['lr']:.2e} "- + " ".join(additional_info)- )-- # Reset trackers- trackers = defaultdict(list)-- start_time = time.time()-- if global_step % cfg.schedule.save_interval == 0:- fabric.save(- Path(cfg.paths.checkpoint_dir) / f"step_{global_step}.ckpt",- {- "model": model,- "optimizer": optimizer,- "scheduler": scheduler.state_dict(),- "global_step": global_step,- },- )-- if (- getattr(cfg.schedule, "eval_interval", None) is not None- and global_step % cfg.schedule.eval_interval == 0- and valid_dataloader is not None- ):- valid(model, valid_dataloader, global_step, fabric, cfg)-- if global_step >= cfg.schedule.max_steps:- break-- last_batch_time = time.time()+ train_metrics = trainer.callback_metrics++ if cfg.get("test"):+ log.info("Starting testing!")+ ckpt_path = trainer.checkpoint_callback.best_model_path+ if ckpt_path == "":+ log.warning("Best ckpt not found! Using current weights for testing...")+ ckpt_path = cfg.get("ckpt_path")++ trainer.test(model=model, datamodule=datamodule, ckpt_path=ckpt_path)+ log.info(f"Best ckpt path: {ckpt_path}")++ test_metrics = trainer.callback_metrics++ # merge train and test metrics+ metric_dict = {**train_metrics, **test_metrics}++ return metric_dict, object_dict@hydra.main(version_base="1.3", config_path="./configs", config_name="llama_pretrain.yaml")-def main(cfg: DictConfig):- log.info(f"Config: \n{OmegaConf.to_yaml(cfg)}")-- if is_flash_attn_available() is False:- log.warning("Flash attention is not available, using default attention")-- fabric: Fabric = hydra.utils.instantiate(cfg.trainer)- fabric.launch()- log.info(f"Fabric: {fabric}")-- model = hydra.utils.instantiate(cfg.model)- log.info(f"Model: {repr(model)}")-- trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)- freeze_params = sum(p.numel() for p in model.parameters() if not p.requires_grad)- log.info(f"Trainable parameters: {trainable_params/1e6:.2f}M")- log.info(f"Freeze parameters: {freeze_params/1e6:.2f}M")-- optimizer = hydra.utils.instantiate(cfg.optimizer, params=model.parameters())- scheduler = hydra.utils.instantiate(cfg.scheduler, optimizer=optimizer)- log.info(f"Optimizer: {optimizer}")- log.info(f"Scheduler: {scheduler}")-- log.info(f"Setup fabric model & dataset")- model = fabric.setup_module(model)- optimizer = fabric.setup_optimizers(optimizer)-- # Build state- global_step = 0-- # Restore training from checkpoint- checkpoint_dir = Path(cfg.paths.checkpoint_dir)- checkpoint_dir.mkdir(parents=True, exist_ok=True)-- # Alphabetically sort checkpoints- checkpoints = natsorted(checkpoint_dir.glob("*.ckpt"))- if len(checkpoints) > 0:- checkpoint_path = checkpoints[-1]-- log.info(f"Restoring checkpoint from {checkpoint_path}")- remainder = fabric.load(- checkpoint_path,- {- "model": model,- "optimizer": optimizer,- "scheduler": scheduler,- },- )- global_step = remainder["global_step"]- log.info(f"Restored global step: {global_step}")-- train_dataloader = hydra.utils.instantiate(cfg.train_dataloader)- log.info(f"Train Dataloader: {train_dataloader}")-- valid_dataloader = None- if getattr(cfg, "valid_dataloader", None) is not None:- valid_dataloader = hydra.utils.instantiate(cfg.valid_dataloader)- log.info(f"Valid Dataloader: {valid_dataloader}")-- train_dataloader = fabric.setup_dataloaders(train_dataloader)- if valid_dataloader is not None:- valid_dataloader = fabric.setup_dataloaders(valid_dataloader)-- log.info(f"Begin training")-- train(- model=model,- optimizer=optimizer,- scheduler=scheduler,- train_dataloader=train_dataloader,- valid_dataloader=valid_dataloader,- global_step=global_step,- fabric=fabric,- cfg=cfg,- )+def main(cfg: DictConfig) -> Optional[float]:+ # train the model+ train(cfg)if __name__ == "__main__":diff --git a/fish_speech/utils/__init__.py b/fish_speech/utils/__init__.pynew file mode 100644index 0000000..c3e0106--- /dev/null+++ b/fish_speech/utils/__init__.py@@ -0,0 +1,21 @@+from .braceexpand import braceexpand+from .file import get_latest_checkpoint+from .instantiators import instantiate_callbacks, instantiate_loggers+from .logger import RankedLogger+from .logging_utils import log_hyperparameters+from .rich_utils import enforce_tags, print_config_tree+from .utils import extras, get_metric_value, task_wrapper++__all__ = [+ "enforce_tags",+ "extras",+ "get_metric_value",+ "RankedLogger",+ "instantiate_callbacks",+ "instantiate_loggers",+ "log_hyperparameters",+ "print_config_tree",+ "task_wrapper",+ "braceexpand",+ "get_latest_checkpoint",+]diff --git a/fish_speech/utils/file.py b/fish_speech/utils/file.pynew file mode 100644index 0000000..dc650d1--- /dev/null+++ b/fish_speech/utils/file.py@@ -0,0 +1,74 @@+import os+from pathlib import Path+from typing import Union++AUDIO_EXTENSIONS = {+ ".mp3",+ ".wav",+ ".flac",+ ".ogg",+ ".m4a",+ ".wma",+ ".aac",+ ".aiff",+ ".aif",+ ".aifc",+}+++def list_files(+ path: Union[Path, str],+ extensions: set[str] = None,+ recursive: bool = False,+ sort: bool = True,+) -> list[Path]:+ """List files in a directory.++ Args:+ path (Path): Path to the directory.+ extensions (set, optional): Extensions to filter. Defaults to None.+ recursive (bool, optional): Whether to search recursively. Defaults to False.+ sort (bool, optional): Whether to sort the files. Defaults to True.++ Returns:+ list: List of files.+ """++ if isinstance(path, str):+ path = Path(path)++ if not path.exists():+ raise FileNotFoundError(f"Directory {path} does not exist.")++ files = (+ [+ Path(os.path.join(root, filename))+ for root, _, filenames in os.walk(path, followlinks=True)+ for filename in filenames+ if Path(os.path.join(root, filename)).is_file()+ ]+ if recursive+ else [f for f in path.glob("*") if f.is_file()]+ )++ if extensions is not None:+ files = [f for f in files if f.suffix in extensions]++ if sort:+ files = sorted(files)++ return files+++def get_latest_checkpoint(path: Path | str) -> Path | None:+ # Find the latest checkpoint+ ckpt_dir = Path(path)++ if ckpt_dir.exists() is False:+ return None++ ckpts = sorted(ckpt_dir.glob("*.ckpt"), key=os.path.getmtime)+ if len(ckpts) == 0:+ return None++ return ckpts[-1]diff --git a/fish_speech/utils/instantiators.py b/fish_speech/utils/instantiators.pynew file mode 100644index 0000000..f6ee463--- /dev/null+++ b/fish_speech/utils/instantiators.py@@ -0,0 +1,50 @@+from typing import List++import hydra+from omegaconf import DictConfig+from pytorch_lightning import Callback+from pytorch_lightning.loggers import Logger++from .logger import RankedLogger++log = RankedLogger(__name__, rank_zero_only=True)+++def instantiate_callbacks(callbacks_cfg: DictConfig) -> List[Callback]:+ """Instantiates callbacks from config."""++ callbacks: List[Callback] = []++ if not callbacks_cfg:+ log.warning("No callback configs found! Skipping..")+ return callbacks++ if not isinstance(callbacks_cfg, DictConfig):+ raise TypeError("Callbacks config must be a DictConfig!")++ for _, cb_conf in callbacks_cfg.items():+ if isinstance(cb_conf, DictConfig) and "_target_" in cb_conf:+ log.info(f"Instantiating callback <{cb_conf._target_}>")+ callbacks.append(hydra.utils.instantiate(cb_conf))++ return callbacks+++def instantiate_loggers(logger_cfg: DictConfig) -> List[Logger]:+ """Instantiates loggers from config."""++ logger: List[Logger] = []++ if not logger_cfg:+ log.warning("No logger configs found! Skipping...")+ return logger++ if not isinstance(logger_cfg, DictConfig):+ raise TypeError("Logger config must be a DictConfig!")++ for _, lg_conf in logger_cfg.items():+ if isinstance(lg_conf, DictConfig) and "_target_" in lg_conf:+ log.info(f"Instantiating logger <{lg_conf._target_}>")+ logger.append(hydra.utils.instantiate(lg_conf))++ return loggerdiff --git a/fish_speech/logger.py b/fish_speech/utils/logger.pysimilarity index 98%rename from fish_speech/logger.pyrename to fish_speech/utils/logger.pyindex 31a76c3..94f94f7 100644--- a/fish_speech/logger.py+++ b/fish_speech/utils/logger.py@@ -10,7 +10,7 @@ class RankedLogger(logging.LoggerAdapter):def __init__(self,name: str = __name__,- rank_zero_only: bool = False,+ rank_zero_only: bool = True,extra: Optional[Mapping[str, object]] = None,) -> None:"""Initializes a multi-GPU-friendly python command line logger that logs on all processesdiff --git a/fish_speech/utils/logging_utils.py b/fish_speech/utils/logging_utils.pynew file mode 100644index 0000000..8e3b0a2--- /dev/null+++ b/fish_speech/utils/logging_utils.py@@ -0,0 +1,48 @@+from lightning.pytorch.utilities import rank_zero_only++from fish_speech.utils import logger as log+++@rank_zero_only+def log_hyperparameters(object_dict: dict) -> None:+ """Controls which config parts are saved by lightning loggers.++ Additionally saves:+ - Number of model parameters+ """++ hparams = {}++ cfg = object_dict["cfg"]+ model = object_dict["model"]+ trainer = object_dict["trainer"]++ if not trainer.logger:+ log.warning("Logger not found! Skipping hyperparameter logging...")+ return++ hparams["model"] = cfg["model"]++ # save number of model parameters+ hparams["model/params/total"] = sum(p.numel() for p in model.parameters())+ hparams["model/params/trainable"] = sum(+ p.numel() for p in model.parameters() if p.requires_grad+ )+ hparams["model/params/non_trainable"] = sum(+ p.numel() for p in model.parameters() if not p.requires_grad+ )++ hparams["data"] = cfg["data"]+ hparams["trainer"] = cfg["trainer"]++ hparams["callbacks"] = cfg.get("callbacks")+ hparams["extras"] = cfg.get("extras")++ hparams["task_name"] = cfg.get("task_name")+ hparams["tags"] = cfg.get("tags")+ hparams["ckpt_path"] = cfg.get("ckpt_path")+ hparams["seed"] = cfg.get("seed")++ # send hparams to all loggers+ for logger in trainer.loggers:+ logger.log_hyperparams(hparams)diff --git a/fish_speech/utils/rich_utils.py b/fish_speech/utils/rich_utils.pynew file mode 100644index 0000000..c8f77c7--- /dev/null+++ b/fish_speech/utils/rich_utils.py@@ -0,0 +1,96 @@+from pathlib import Path+from typing import Sequence++import rich+import rich.syntax+import rich.tree+from hydra.core.hydra_config import HydraConfig+from lightning.pytorch.utilities import rank_zero_only+from omegaconf import DictConfig, OmegaConf, open_dict+from rich.prompt import Prompt++from fish_speech.utils import logger as log+++@rank_zero_only+def print_config_tree(+ cfg: DictConfig,+ print_order: Sequence[str] = (+ "data",+ "model",+ "callbacks",+ "logger",+ "trainer",+ "paths",+ "extras",+ ),+ resolve: bool = False,+ save_to_file: bool = False,+) -> None:+ """Prints content of DictConfig using Rich library and its tree structure.++ Args:+ cfg (DictConfig): Configuration composed by Hydra.+ print_order (Sequence[str], optional): Determines in what order config components are printed.+ resolve (bool, optional): Whether to resolve reference fields of DictConfig.+ save_to_file (bool, optional): Whether to export config to the hydra output folder.+ """ # noqa: E501++ style = "dim"+ tree = rich.tree.Tree("CONFIG", style=style, guide_style=style)++ queue = []++ # add fields from `print_order` to queue+ for field in print_order:+ queue.append(field) if field in cfg else log.warning(+ f"Field '{field}' not found in config. "+ + f"Skipping '{field}' config printing..."+ )++ # add all the other fields to queue (not specified in `print_order`)+ for field in cfg:+ if field not in queue:+ queue.append(field)++ # generate config tree from queue+ for field in queue:+ branch = tree.add(field, style=style, guide_style=style)++ config_group = cfg[field]+ if isinstance(config_group, DictConfig):+ branch_content = OmegaConf.to_yaml(config_group, resolve=resolve)+ else:+ branch_content = str(config_group)++ branch.add(rich.syntax.Syntax(branch_content, "yaml"))++ # print config tree+ rich.print(tree)++ # save config tree to file+ if save_to_file:+ with open(Path(cfg.paths.output_dir, "config_tree.log"), "w") as file:+ rich.print(tree, file=file)+++@rank_zero_only+def enforce_tags(cfg: DictConfig, save_to_file: bool = False) -> None:+ """Prompts user to input tags from command line if no tags are provided in config.""" # noqa: E501++ if not cfg.get("tags"):+ if "id" in HydraConfig().cfg.hydra.job:+ raise ValueError("Specify tags before launching a multirun!")++ log.warning("No tags provided in config. Prompting user to input tags...")+ tags = Prompt.ask("Enter a list of comma separated tags", default="dev")+ tags = [t.strip() for t in tags.split(",") if t != ""]++ with open_dict(cfg):+ cfg.tags = tags++ log.info(f"Tags: {cfg.tags}")++ if save_to_file:+ with open(Path(cfg.paths.output_dir, "tags.log"), "w") as file:+ rich.print(cfg.tags, file=file)diff --git a/fish_speech/utils/utils.py b/fish_speech/utils/utils.pynew file mode 100644index 0000000..c546bfa--- /dev/null+++ b/fish_speech/utils/utils.py@@ -0,0 +1,114 @@+import warnings+from importlib.util import find_spec+from typing import Callable++from omegaconf import DictConfig++from .logger import RankedLogger+from .rich_utils import enforce_tags, print_config_tree++log = RankedLogger(__name__, rank_zero_only=True)+++def extras(cfg: DictConfig) -> None:+ """Applies optional utilities before the task is started.++ Utilities:+ - Ignoring python warnings+ - Setting tags from command line+ - Rich config printing+ """++ # return if no `extras` config+ if not cfg.get("extras"):+ log.warning("Extras config not found! <cfg.extras=null>")+ return++ # disable python warnings+ if cfg.extras.get("ignore_warnings"):+ log.info("Disabling python warnings! <cfg.extras.ignore_warnings=True>")+ warnings.filterwarnings("ignore")++ # prompt user to input tags from command line if none are provided in the config+ if cfg.extras.get("enforce_tags"):+ log.info("Enforcing tags! <cfg.extras.enforce_tags=True>")+ enforce_tags(cfg, save_to_file=True)++ # pretty print config tree using Rich library+ if cfg.extras.get("print_config"):+ log.info("Printing config tree with Rich! <cfg.extras.print_config=True>")+ print_config_tree(cfg, resolve=True, save_to_file=True)+++def task_wrapper(task_func: Callable) -> Callable:+ """Optional decorator that controls the failure behavior when executing the task function.++ This wrapper can be used to:+ - make sure loggers are closed even if the task function raises an exception (prevents multirun failure)+ - save the exception to a `.log` file+ - mark the run as failed with a dedicated file in the `logs/` folder (so we can find and rerun it later)+ - etc. (adjust depending on your needs)++ Example:+ ```+ @utils.task_wrapper+ def train(cfg: DictConfig) -> Tuple[dict, dict]:++ ...++ return metric_dict, object_dict+ ```+ """ # noqa: E501++ def wrap(cfg: DictConfig):+ # execute the task+ try:+ metric_dict, object_dict = task_func(cfg=cfg)++ # things to do if exception occurs+ except Exception as ex:+ # save exception to `.log` file+ log.exception("")++ # some hyperparameter combinations might be invalid or+ # cause out-of-memory errors so when using hparam search+ # plugins like Optuna, you might want to disable+ # raising the below exception to avoid multirun failure+ raise ex++ # things to always do after either success or exception+ finally:+ # display output dir path in terminal+ log.info(f"Output dir: {cfg.paths.run_dir}")++ # always close wandb run (even if exception occurs so multirun won't fail)+ if find_spec("wandb"): # check if wandb is installed+ import wandb++ if wandb.run:+ log.info("Closing wandb!")+ wandb.finish()++ return metric_dict, object_dict++ return wrap+++def get_metric_value(metric_dict: dict, metric_name: str) -> float:+ """Safely retrieves value of the metric logged in LightningModule."""++ if not metric_name:+ log.info("Metric name is None! Skipping metric value retrieval...")+ return None++ if metric_name not in metric_dict:+ raise Exception(+ f"Metric value not found! <metric_name={metric_name}>\n"+ "Make sure metric name logged in LightningModule is correct!\n"+ "Make sure `optimized_metric` name in `hparams_search` config is correct!"+ )++ metric_value = metric_dict[metric_name].item()+ log.info(f"Retrieved metric value! <{metric_name}={metric_value}>")++ return metric_valuediff --git a/fish_speech/utils/viz.py b/fish_speech/utils/viz.pynew file mode 100644index 0000000..073cc03--- /dev/null+++ b/fish_speech/utils/viz.py@@ -0,0 +1,29 @@+import matplotlib+from matplotlib import pyplot as plt+from torch import Tensor++matplotlib.use("Agg")+++def plot_mel(data, titles=None):+ fig, axes = plt.subplots(len(data), 1, squeeze=False)++ if titles is None:+ titles = [None for i in range(len(data))]++ plt.tight_layout()++ for i in range(len(data)):+ mel = data[i]++ if isinstance(mel, Tensor):+ mel = mel.detach().cpu().numpy()++ axes[i][0].imshow(mel, origin="lower")+ axes[i][0].set_aspect(2.5, adjustable="box")+ axes[i][0].set_ylim(0, mel.shape[0])+ axes[i][0].set_title(titles[i], fontsize="medium")+ axes[i][0].tick_params(labelsize="x-small", left=False, labelleft=False)+ axes[i][0].set_anchor("W")++ return figdiff --git a/pyrightconfig.json b/pyrightconfig.jsonnew file mode 100644index 0000000..d72b55e--- /dev/null+++ b/pyrightconfig.json@@ -0,0 +1,6 @@+{+ "exclude": [+ "data",+ "filelists"+ ]+}\ No newline at end of filediff --git a/tools/build_vq_text.py b/tools/build_vq_text.pynew file mode 100644index 0000000..ce7c5a9--- /dev/null+++ b/tools/build_vq_text.py@@ -0,0 +1,32 @@+from pathlib import Path++from datasets import Dataset+++def parse_data(wav_dir, item):+ text_file = (wav_dir / item["item_name"]).with_suffix(".txt")+ text = text_file.read_text().strip()++ semantic = item["semantic_audio"]+ semantic = [f"<semantic_{x}>" for x in semantic.split(" ")]+ semantic = " ".join(semantic)++ text = f"[INST] {text} [/INST] {semantic} </s>"++ return {+ "text": text,+ }+++if __name__ == "__main__":+ # dataset = WenetVQDataset()+ # dataset = list(dataset)+ # print("Initialized dataset.")+ dataset = Dataset.from_csv("data/cn-hubert-wenet-25hz-semantic.tsv", delimiter="\t")+ dataset = dataset.map(+ lambda item: parse_data(Path("data/WenetSpeech"), item), num_proc=64+ )+ dataset = dataset.remove_columns(["item_name", "semantic_audio"])+ dataset = dataset.train_test_split(test_size=0.01)+ print(dataset["test"][0])+ dataset.push_to_hub("fishaudio/wenet-vq", private=True)