Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

DDP for multiple cards (#1897)

Yi Liu33b7df0ead30
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index ed627eb5a..08c089877 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -210,7 +210,7 @@ class SignRoundQuantizer(RTNLayerFallbackMixin, BaseQuantizer):
if self.gradient_accumulate_steps != 1 and not self.attention_mask:
whole_indices = torch.arange(global_batch_size)
num_elm = ctx.count_batch_elements(whole_indices)
- setup_ddp_if_needed_(self, block, device_manager.device_list)
+ block, sync_gradients = setup_ddp_if_needed_(self, block, device_manager.device_list)
index_sampler = IndexSampler(nsamples, global_batch_size)
batch_size = self.batch_size
for i in range(self.iters):
@@ -256,6 +256,7 @@ class SignRoundQuantizer(RTNLayerFallbackMixin, BaseQuantizer):
if not self.not_use_best_mse:
if 0 < self.dynamic_max_gap <= i - last_best_iter:
break
+ sync_gradients()
self._step(scaler, optimizer, lr_schedule)
last_loss = total_loss
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index c79e6980d..5364a8280 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -1291,7 +1291,8 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, None
parsed.append(f"{p}:0")
else:
parsed.append(p)
- return list(set(parsed))
+ # Preserve user-specified device order while removing duplicates.
+ return list(dict.fromkeys(parsed))
if isinstance(device_map, dict):
# Extract all devices recursively from dict values
diff --git a/auto_round/utils/distributed.py b/auto_round/utils/distributed.py
index 47f12ba29..afe5a04ed 100644
--- a/auto_round/utils/distributed.py
+++ b/auto_round/utils/distributed.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
from functools import lru_cache
import torch
@@ -27,20 +28,97 @@ def is_distributed():
def setup_ddp_if_needed_(ar, block: torch.nn.Module, device_list: list[int]):
- """Wrap ``block`` with DDP when distributed execution is enabled.
+ """Prepare ``block`` for distributed training and return a gradient-sync hook.
- This experimental hook coordinates AutoRound with the
- LLM-Compressor distributed workflow for DDP quantization.
+ Returns ``(block, sync_fn)`` where ``sync_fn()`` must be called after gradient
+ accumulation and before ``optimizer.step()``.
+
+ * **Non-distributed**: returns ``(block, noop)``.
+ * **Single GPU per rank**: wraps ``block`` with DDP, which synchronizes
+ gradients automatically during the backward pass. ``sync_fn`` is a no-op.
+ * **Multi-GPU per rank**: the block's submodules are sharded across GPUs
+ and cannot be DDP-wrapped. ``sync_fn`` performs a manual ``all_reduce``
+ (AVG) on every parameter that has a gradient, so that each rank sees the
+ same averaged gradient before taking a step.
+
+ .. note::
+ ``ReduceOp.AVG`` is safe for SignSGD because
+ ``sign(avg(g)) == sign(sum(g))`` for any non-zero sum.
Args:
ar: AutoRound instance invoking the helper.
block: Model block that may need DDP wrapping.
- device_list: Device identifiers to pass through to DDP.
+ device_list: Device identifiers assigned to the current rank.
+
+ Returns:
+ ``(block, sync_fn)``
"""
+ import torch.distributed as dist
+
+ visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "unset")
+
if not is_distributed():
- return
- from torch.nn.parallel import DistributedDataParallel as DDP
+ return block, _noop_sync
+
+ num_devices = len(device_list)
+
+ if num_devices == 1:
+ from torch.nn.parallel import DistributedDataParallel as DDP
+
+ logger.warning_once("AutoRound DDP is an experimental feature, please use with caution.")
+ logger.trace(
+ "[Rank: %d] Wrapping block with DDP on device_list=%s, CUDA_VISIBLE_DEVICES=%s",
+ dist.get_rank(),
+ device_list,
+ visible_devices,
+ )
+ # Ensure all block parameters are on the DDP device before wrapping.
+ _move_block_to_device(block, device_list[0])
+ block = DDP(block, device_ids=device_list, find_unused_parameters=True)
+ return block, _noop_sync
+
+ # Multi-GPU per rank: block is sharded, can't DDP-wrap.
+ # Use manual all_reduce on gradients before optimizer step.
+ logger.warning_once("AutoRound multi-GPU DDP is an experimental feature, " "please use with caution.")
+ logger.trace(
+ (
+ "[Rank: %d] Multi-GPU(device_list=%s, CUDA_VISIBLE_DEVICES=%s) per rank (%d GPUs),"
+ " using manual all_reduce sync"
+ ),
+ dist.get_rank(),
+ str(device_list),
+ visible_devices,
+ num_devices,
+ )
+
+ def _sync_gradients():
+ _all_reduce_model_grads(block)
+
+ return block, _sync_gradients
+
+
+def _noop_sync():
+ pass
+
+
+def _move_block_to_device(module: torch.nn.Module, device):
+ """Move *module* to *device*, used before DDP wrapping."""
+ module.to(device)
+
+
+def _all_reduce_model_grads(module: torch.nn.Module):
+ """All-reduce (AVG) gradients of all parameters in *module* across ranks."""
+ import torch.distributed as dist
- logger.warning_once("AutoRound DDP is an experimental feature, please use with caution.")
- logger.trace("[Rank: %d] Using device list: %s", torch.distributed.get_rank(), device_list)
- block = DDP(block, device_ids=device_list, find_unused_parameters=True)
+ comm_device = torch.cuda.current_device() if torch.cuda.is_available() else None
+ if comm_device is not None:
+ comm_device = torch.device("cuda", comm_device)
+ for param in module.parameters():
+ if param.grad is not None:
+ grad = param.grad
+ if comm_device is not None and grad.is_cuda and grad.device != comm_device:
+ synced = grad.to(comm_device)
+ dist.all_reduce(synced, op=dist.ReduceOp.AVG)
+ grad.copy_(synced.to(grad.device))
+ else:
+ dist.all_reduce(grad, op=dist.ReduceOp.AVG)