Skip to content

SOTA Open Source TTS

fix: correct progress bar total when using gradient accumulation with max_steps (#1227)

Matteo48b19f66b1df
diff --git a/fish_speech/callbacks/__init__.py b/fish_speech/callbacks/__init__.py
index bbcf3f3..375c443 100644
--- a/fish_speech/callbacks/__init__.py
+++ b/fish_speech/callbacks/__init__.py
@@ -1,3 +1,4 @@
from .grad_norm import GradNormMonitor
+from .progress_bar import GradAccumProgressBar
-__all__ = ["GradNormMonitor"]
+__all__ = ["GradNormMonitor", "GradAccumProgressBar"]
diff --git a/fish_speech/callbacks/progress_bar.py b/fish_speech/callbacks/progress_bar.py
new file mode 100644
index 0000000..2a56615
--- /dev/null
+++ b/fish_speech/callbacks/progress_bar.py
@@ -0,0 +1,16 @@
+from lightning.pytorch.callbacks import TQDMProgressBar
+
+
+class GradAccumProgressBar(TQDMProgressBar):
+ """
+ Progress bar that accounts for gradient accumulation so the total
+ reflects actual forward passes rather than optimizer steps.
+ """
+
+ @property
+ def total_train_batches(self):
+ total = super().total_train_batches
+ accumulate = self.trainer.accumulate_grad_batches
+ if isinstance(total, int) and accumulate > 1:
+ return total * accumulate
+ return total
diff --git a/fish_speech/configs/base.yaml b/fish_speech/configs/base.yaml
index 99e6dab..9073b5d 100644
--- a/fish_speech/configs/base.yaml
+++ b/fish_speech/configs/base.yaml
@@ -57,6 +57,9 @@ callbacks:
norm_type: 2
logging_interval: step
+ progress_bar:
+ _target_: fish_speech.callbacks.GradAccumProgressBar
+
# Logger
logger:
tensorboard: