PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
DDP for multiple cards (#1897)
Yi Liu33b7df0ead30
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex ed627eb5a..08c089877 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -210,7 +210,7 @@ class SignRoundQuantizer(RTNLayerFallbackMixin, BaseQuantizer):if self.gradient_accumulate_steps != 1 and not self.attention_mask:whole_indices = torch.arange(global_batch_size)num_elm = ctx.count_batch_elements(whole_indices)- setup_ddp_if_needed_(self, block, device_manager.device_list)+ block, sync_gradients = setup_ddp_if_needed_(self, block, device_manager.device_list)index_sampler = IndexSampler(nsamples, global_batch_size)batch_size = self.batch_sizefor i in range(self.iters):@@ -256,6 +256,7 @@ class SignRoundQuantizer(RTNLayerFallbackMixin, BaseQuantizer):if not self.not_use_best_mse:if 0 < self.dynamic_max_gap <= i - last_best_iter:break+ sync_gradients()self._step(scaler, optimizer, lr_schedule)last_loss = total_lossdiff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex c79e6980d..5364a8280 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -1291,7 +1291,8 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, Noneparsed.append(f"{p}:0")else:parsed.append(p)- return list(set(parsed))+ # Preserve user-specified device order while removing duplicates.+ return list(dict.fromkeys(parsed))if isinstance(device_map, dict):# Extract all devices recursively from dict valuesdiff --git a/auto_round/utils/distributed.py b/auto_round/utils/distributed.pyindex 47f12ba29..afe5a04ed 100644--- a/auto_round/utils/distributed.py+++ b/auto_round/utils/distributed.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import osfrom functools import lru_cacheimport torch@@ -27,20 +28,97 @@ def is_distributed():def setup_ddp_if_needed_(ar, block: torch.nn.Module, device_list: list[int]):- """Wrap ``block`` with DDP when distributed execution is enabled.+ """Prepare ``block`` for distributed training and return a gradient-sync hook.- This experimental hook coordinates AutoRound with the- LLM-Compressor distributed workflow for DDP quantization.+ Returns ``(block, sync_fn)`` where ``sync_fn()`` must be called after gradient+ accumulation and before ``optimizer.step()``.++ * **Non-distributed**: returns ``(block, noop)``.+ * **Single GPU per rank**: wraps ``block`` with DDP, which synchronizes+ gradients automatically during the backward pass. ``sync_fn`` is a no-op.+ * **Multi-GPU per rank**: the block's submodules are sharded across GPUs+ and cannot be DDP-wrapped. ``sync_fn`` performs a manual ``all_reduce``+ (AVG) on every parameter that has a gradient, so that each rank sees the+ same averaged gradient before taking a step.++ .. note::+ ``ReduceOp.AVG`` is safe for SignSGD because+ ``sign(avg(g)) == sign(sum(g))`` for any non-zero sum.Args:ar: AutoRound instance invoking the helper.block: Model block that may need DDP wrapping.- device_list: Device identifiers to pass through to DDP.+ device_list: Device identifiers assigned to the current rank.++ Returns:+ ``(block, sync_fn)``"""+ import torch.distributed as dist++ visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "unset")+if not is_distributed():- return- from torch.nn.parallel import DistributedDataParallel as DDP+ return block, _noop_sync++ num_devices = len(device_list)++ if num_devices == 1:+ from torch.nn.parallel import DistributedDataParallel as DDP++ logger.warning_once("AutoRound DDP is an experimental feature, please use with caution.")+ logger.trace(+ "[Rank: %d] Wrapping block with DDP on device_list=%s, CUDA_VISIBLE_DEVICES=%s",+ dist.get_rank(),+ device_list,+ visible_devices,+ )+ # Ensure all block parameters are on the DDP device before wrapping.+ _move_block_to_device(block, device_list[0])+ block = DDP(block, device_ids=device_list, find_unused_parameters=True)+ return block, _noop_sync++ # Multi-GPU per rank: block is sharded, can't DDP-wrap.+ # Use manual all_reduce on gradients before optimizer step.+ logger.warning_once("AutoRound multi-GPU DDP is an experimental feature, " "please use with caution.")+ logger.trace(+ (+ "[Rank: %d] Multi-GPU(device_list=%s, CUDA_VISIBLE_DEVICES=%s) per rank (%d GPUs),"+ " using manual all_reduce sync"+ ),+ dist.get_rank(),+ str(device_list),+ visible_devices,+ num_devices,+ )++ def _sync_gradients():+ _all_reduce_model_grads(block)++ return block, _sync_gradients+++def _noop_sync():+ pass+++def _move_block_to_device(module: torch.nn.Module, device):+ """Move *module* to *device*, used before DDP wrapping."""+ module.to(device)+++def _all_reduce_model_grads(module: torch.nn.Module):+ """All-reduce (AVG) gradients of all parameters in *module* across ranks."""+ import torch.distributed as dist- logger.warning_once("AutoRound DDP is an experimental feature, please use with caution.")- logger.trace("[Rank: %d] Using device list: %s", torch.distributed.get_rank(), device_list)- block = DDP(block, device_ids=device_list, find_unused_parameters=True)+ comm_device = torch.cuda.current_device() if torch.cuda.is_available() else None+ if comm_device is not None:+ comm_device = torch.device("cuda", comm_device)+ for param in module.parameters():+ if param.grad is not None:+ grad = param.grad+ if comm_device is not None and grad.is_cuda and grad.device != comm_device:+ synced = grad.to(comm_device)+ dist.all_reduce(synced, op=dist.ReduceOp.AVG)+ grad.copy_(synced.to(grad.device))+ else:+ dist.all_reduce(grad, op=dist.ReduceOp.AVG)