Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

8.2 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
"""Unit tests for ``auto_round.utils.distributed``."""

from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest
import torch
import torch.nn as nn

from auto_round.utils.distributed import (
    _all_reduce_model_grads,
    _move_block_to_device,
    _noop_sync,
    is_distributed,
    setup_ddp_if_needed_,
)


class TestIsDistributed:
    """Test is_distributed with mocked torch.distributed."""

    def test_not_initialized(self):
        with (
            patch("auto_round.utils.distributed.is_distributed", return_value=False),
            patch("torch.distributed.is_initialized", return_value=False),
        ):
            # Clear cache to ensure fresh evaluation
            is_distributed.cache_clear()
            result = is_distributed()
            assert result is False

    def test_initialized_single_device(self):
        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=1),
        ):
            is_distributed.cache_clear()
            result = is_distributed()
            assert result is False

    def test_initialized_multi_device(self):
        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=4),
        ):
            is_distributed.cache_clear()
            result = is_distributed()
            assert result is True

    def test_dist_not_initialized(self):
        with patch("torch.distributed.is_initialized", return_value=False):
            is_distributed.cache_clear()
            assert is_distributed() is False
            is_distributed.cache_clear()

    def test_dist_initialized_single_world(self):
        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=1),
        ):
            is_distributed.cache_clear()
            assert is_distributed() is False
            is_distributed.cache_clear()

    def test_dist_initialized_multi_world(self):
        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=2),
        ):
            is_distributed.cache_clear()
            assert is_distributed() is True
            is_distributed.cache_clear()


class TestNoopSync:
    """Test _noop_sync."""

    def test_noop_sync_does_nothing(self):
        # Should not raise
        _noop_sync()

    def test_returns_none(self):
        assert _noop_sync() is None


class TestMoveBlockToDevice:
    """Test _move_block_to_device."""

    def test_moves_to_cpu(self):
        block = nn.Linear(4, 4)
        if torch.cuda.is_available():
            block = block.to("cuda")
        _move_block_to_device(block, "cpu")
        assert next(block.parameters()).device.type == "cpu"

    def test_moves_to_cuda(self):
        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")
        block = nn.Linear(4, 4)
        _move_block_to_device(block, 0)
        assert next(block.parameters()).device.type == "cuda"

    def test_moves_to_device(self):
        layer = nn.Linear(4, 4)
        _move_block_to_device(layer, "cpu")
        # Should be on cpu
        assert next(layer.parameters()).device == torch.device("cpu")


class TestAllReduceModelGrads:
    """Test _all_reduce_model_grads."""

    def test_no_grad_is_noop(self):
        model = nn.Linear(4, 4)
        # No gradients set - should not raise
        _all_reduce_model_grads(model)

    def test_no_grads_does_nothing(self):
        layer = nn.Linear(4, 4)
        # No grads set, should not raise
        _all_reduce_model_grads(layer)

    def test_no_distributed_raises(self):
        """Without distributed initialized, all_reduce is a no-op."""
        model = nn.Linear(4, 4)
        model.weight.grad = torch.randn_like(model.weight)
        _all_reduce_model_grads(model)

    def test_raises_when_dist_not_initialized(self):
        layer = nn.Linear(4, 4)
        layer.weight.grad = torch.randn_like(layer.weight)
        with patch("torch.cuda.is_available", return_value=False):
            _all_reduce_model_grads(layer)

    def test_with_cuda_grad_no_distributed(self):
        """Grad is CUDA but distributed not initialized - should not raise."""
        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")
        model = nn.Linear(4, 4).to("cuda")
        model.weight.grad = torch.randn_like(model.weight)
        _all_reduce_model_grads(model)


class TestSetupDdpIfNeeded:
    """Test setup_ddp_if_needed_."""

    def test_non_distributed_returns_block_noop(self):
        model = nn.Linear(4, 4)
        ar = SimpleNamespace()
        with patch("auto_round.utils.distributed.is_distributed", return_value=False):
            block, sync_fn = setup_ddp_if_needed_(ar, model, [0])
            assert block is model
            assert sync_fn is _noop_sync

    def test_non_distributed_respects_device_list(self):
        """Device list is passed but distributed is off, so DDP not used."""
        model = nn.Linear(4, 4)
        ar = SimpleNamespace()
        with patch("auto_round.utils.distributed.is_distributed", return_value=False):
            block, sync_fn = setup_ddp_if_needed_(ar, model, [0, 1])
            assert block is model
            assert sync_fn is _noop_sync

    def test_single_device_ddp(self):
        """Distributed with single GPU per rank wraps with DDP."""
        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")
        model = nn.Linear(4, 4).to("cpu")

        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=2),
            patch("torch.distributed.get_rank", return_value=0),
            patch("torch.nn.parallel.DistributedDataParallel"),
        ):
            ar = SimpleNamespace()
            block, sync_fn = setup_ddp_if_needed_(ar, model, [0])
            assert sync_fn is _noop_sync

    def test_single_device_returns_noop_when_distributed(self):
        """Test the multi-GPU case which doesn't need to move to GPU device."""
        with patch("auto_round.utils.distributed.is_distributed", return_value=True):
            block = nn.Linear(4, 4)
            with patch("torch.distributed.get_rank", return_value=0):
                # Use multi-device path which doesn't actually move to GPU
                block, sync_fn = setup_ddp_if_needed_(None, block, [0, 1])
            # Multi-device path returns a manual reduce sync fn
            assert sync_fn is not _noop_sync

    def test_multi_device_uses_manual_reduce(self):
        with patch("auto_round.utils.distributed.is_distributed", return_value=True):
            block = nn.Linear(4, 4)
            with patch("torch.distributed.get_rank", return_value=0):
                block, sync_fn = setup_ddp_if_needed_(None, block, [0, 1])
            # Should return a custom sync function
            assert sync_fn is not _noop_sync
            # Calling sync_fn should call _all_reduce_model_grads
            with patch("auto_round.utils.distributed._all_reduce_model_grads") as mock_reduce:
                sync_fn()
                mock_reduce.assert_called_once()

    def test_multi_device_manual_reduce(self):
        """Distributed with multiple GPUs per rank returns manual sync."""
        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")

        model = nn.Linear(4, 4)

        with (
            patch("torch.distributed.is_initialized", return_value=True),
            patch("torch.distributed.get_world_size", return_value=4),
            patch("torch.distributed.get_rank", return_value=0),
        ):
            ar = SimpleNamespace()
            block, sync_fn = setup_ddp_if_needed_(ar, model, [0, 1])
            # Should not be the noop
            assert sync_fn is not _noop_sync
            # Calling it should not raise
            sync_fn()