SOTA Open Source TTS
fix: correct progress bar total when using gradient accumulation with max_steps (#1227)
Matteo48b19f66b1df
diff --git a/fish_speech/callbacks/__init__.py b/fish_speech/callbacks/__init__.pyindex bbcf3f3..375c443 100644--- a/fish_speech/callbacks/__init__.py+++ b/fish_speech/callbacks/__init__.py@@ -1,3 +1,4 @@from .grad_norm import GradNormMonitor+from .progress_bar import GradAccumProgressBar-__all__ = ["GradNormMonitor"]+__all__ = ["GradNormMonitor", "GradAccumProgressBar"]diff --git a/fish_speech/callbacks/progress_bar.py b/fish_speech/callbacks/progress_bar.pynew file mode 100644index 0000000..2a56615--- /dev/null+++ b/fish_speech/callbacks/progress_bar.py@@ -0,0 +1,16 @@+from lightning.pytorch.callbacks import TQDMProgressBar+++class GradAccumProgressBar(TQDMProgressBar):+ """+ Progress bar that accounts for gradient accumulation so the total+ reflects actual forward passes rather than optimizer steps.+ """++ @property+ def total_train_batches(self):+ total = super().total_train_batches+ accumulate = self.trainer.accumulate_grad_batches+ if isinstance(total, int) and accumulate > 1:+ return total * accumulate+ return totaldiff --git a/fish_speech/configs/base.yaml b/fish_speech/configs/base.yamlindex 99e6dab..9073b5d 100644--- a/fish_speech/configs/base.yaml+++ b/fish_speech/configs/base.yaml@@ -57,6 +57,9 @@ callbacks:norm_type: 2logging_interval: step+ progress_bar:+ _target_: fish_speech.callbacks.GradAccumProgressBar+# Loggerlogger:tensorboard: