Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

perf: optimize training pipeline for ~3% speedup

游雁18a980b0ddfd
diff --git a/funasr/datasets/audio_datasets/espnet_samplers.py b/funasr/datasets/audio_datasets/espnet_samplers.py
index a411ba35f..e05ce22b3 100644
--- a/funasr/datasets/audio_datasets/espnet_samplers.py
+++ b/funasr/datasets/audio_datasets/espnet_samplers.py
@@ -24,6 +24,10 @@ def EspnetStyleBatchSampler_fn(dataset, **kwargs):
dataloader_args["batch_sampler"] = batch_sampler
dataloader_args["num_workers"] = kwargs.get("num_workers", 4)
dataloader_args["pin_memory"] = kwargs.get("pin_memory", True)
+ num_workers = dataloader_args.get("num_workers", 4)
+ if num_workers > 0:
+ dataloader_args["persistent_workers"] = kwargs.get("persistent_workers", True)
+ dataloader_args["prefetch_factor"] = kwargs.get("prefetch_factor", 2)
return dataloader_args
diff --git a/funasr/datasets/audio_datasets/samplers.py b/funasr/datasets/audio_datasets/samplers.py
index f478db002..2cd41c15d 100644
--- a/funasr/datasets/audio_datasets/samplers.py
+++ b/funasr/datasets/audio_datasets/samplers.py
@@ -39,6 +39,10 @@ def CustomDistributedBatchSampler_fn(dataset, **kwargs):
dataloader_args["batch_sampler"] = batch_sampler
dataloader_args["num_workers"] = kwargs.get("num_workers", 4)
dataloader_args["pin_memory"] = kwargs.get("pin_memory", True)
+ num_workers = dataloader_args.get("num_workers", 4)
+ if num_workers > 0:
+ dataloader_args["persistent_workers"] = kwargs.get("persistent_workers", True)
+ dataloader_args["prefetch_factor"] = kwargs.get("prefetch_factor", 2)
return dataloader_args
diff --git a/funasr/train_utils/trainer.py b/funasr/train_utils/trainer.py
index c95acd658..f2815c128 100644
--- a/funasr/train_utils/trainer.py
+++ b/funasr/train_utils/trainer.py
@@ -380,7 +380,7 @@ class Trainer:
time1 = time.perf_counter()
speed_stats["data_load"] = f"{time1-time_beg:0.3f}"
- batch = to_device(batch, self.device)
+ batch = to_device(batch, self.device, non_blocking=True)
my_context = nullcontext
if self.use_ddp or self.use_fsdp:
@@ -461,7 +461,7 @@ class Trainer:
# Clear gradients for the next accumulation stage
optim.zero_grad(set_to_none=True)
- if self.use_ddp or self.use_fsdp:
+ if (self.use_ddp or self.use_fsdp) and (batch_idx + 1) % self.log_interval == 0:
train_loss_avg = torch.tensor(self.train_loss_avg, dtype=torch.float32).to(
self.device
)
@@ -567,7 +567,7 @@ class Trainer:
break
time1 = time.perf_counter()
speed_stats["data_load"] = f"{time1 - time5:0.3f}"
- batch = to_device(batch, self.device)
+ batch = to_device(batch, self.device, non_blocking=True)
time2 = time.perf_counter()
retval = model(**batch)
diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.py
index 3e5f68eee..f4f319d31 100644
--- a/funasr/train_utils/trainer_ds.py
+++ b/funasr/train_utils/trainer_ds.py
@@ -600,7 +600,7 @@ class Trainer:
time1 = time.perf_counter()
loss_dict["speed_stats"]["data_load"] = f"{time1-time_beg:0.3f}"
- batch = to_device(batch, self.device)
+ batch = to_device(batch, self.device, non_blocking=True)
my_context = nullcontext
if self.use_ddp or self.use_fsdp:
@@ -798,7 +798,7 @@ class Trainer:
time1 = time.perf_counter()
loss_dict["speed_stats"]["data_load"] = f"{time1 - time_beg:0.3f}"
- batch = to_device(batch, self.device)
+ batch = to_device(batch, self.device, non_blocking=True)
time2 = time.perf_counter()