Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

6.0 KB
Raw Blame
"""Tests for the one-shot auto linear_loop pick (AR_MOE_EXPERTS_IMPL=auto).

Grouped tuning stacks per-expert fp32 tensors on every expert-homing device;
when any such device cannot hold state + stacks + routed transients beside the
reserve, the first MoE tune block must switch the run to linear_loop BEFORE
the loop starts (instead of OOMing inside grouped_mm and re-staging the sliced
fallback). Explicit AR_MOE_EXPERTS_IMPL choices are never overridden.
"""

import unittest
from unittest import mock

GB = 2**30


def _envs_auto():
    from auto_round import envs

    return mock.patch.object(envs, "AR_MOE_EXPERTS_IMPL", "auto")


class _Cfg:
    def __init__(self):
        self._experts_implementation = "linear_grouped"


class TestAutoLinearLoop(unittest.TestCase):
    def setUp(self):
        import auto_round.algorithms.quantization.tune_memory as q

        self.q = q
        q._MOE_IMPL_AUTO_LINEAR_LOOP_DONE = False
        self.cfg = _Cfg()
        self.state = {"cuda:1": 14 * GB, "cuda:2": 14 * GB}  # 14 B/param layout
        self.act = {"cuda:1": 12 * GB, "cuda:2": 2 * GB}  # stacks dominate cuda:1

    def _run(self, iters=20, stacks=True):
        q = self.q
        with (
            _envs_auto(),
            mock.patch.object(q, "_logical_state_by_device", return_value=self.state),
            mock.patch.object(q, "_activation_bytes_by_device", return_value=self.act),
            mock.patch.object(q, "_grouped_stack_bytes", side_effect=lambda b, d, config=None: 1 if stacks else 0),
            mock.patch("auto_round.utils.device.probe_usable_bytes", return_value=16 * GB),
        ):
            q._maybe_auto_linear_loop_for_tuning(object(), [object()], 8, iters, self.cfg, None)

    def test_over_lane_switches_before_the_loop(self):
        # demand on cuda:1 = 14*6/14 (grads+bf16; snapshot parks) + 12 + 0.25
        # = 20.25 GiB > free 16 (guaranteed in-loop terms only)
        self._run()
        self.assertEqual(self.cfg._experts_implementation, "linear_loop")
        self.assertTrue(self.q._MOE_IMPL_AUTO_LINEAR_LOOP_DONE)

    def test_fits_lane_keeps_grouped(self):
        self.act = {"cuda:1": 1 * GB, "cuda:2": 1 * GB}
        self._run()
        self.assertEqual(self.cfg._experts_implementation, "linear_grouped")

    def test_explicit_impl_never_overridden(self):
        from auto_round import envs

        with mock.patch.object(envs, "AR_MOE_EXPERTS_IMPL", "linear_grouped"):
            self.q._maybe_auto_linear_loop_for_tuning(object(), [], 8, 20, self.cfg, None)
        self.assertEqual(self.cfg._experts_implementation, "linear_grouped")

    def test_iters0_untouched(self):
        self._run(iters=0)
        self.assertEqual(self.cfg._experts_implementation, "linear_grouped")

    def test_model_context_config_without_impl_attr_still_decides(self):
        # server regression: model_context.config never received
        # _experts_implementation (prepare writes it on model.config); the
        # old single-candidate read silently returned and the 4x3090 lane
        # never switched. The check must fall through to model.config, and
        # a switch must update BOTH config objects.
        ctx_cfg = type("CtxCfg", (), {})()  # no _experts_implementation attr
        model = type("M", (), {})()
        model.config = self.cfg
        q = self.q
        with (
            _envs_auto(),
            mock.patch.object(q, "_logical_state_by_device", return_value=self.state),
            mock.patch.object(q, "_activation_bytes_by_device", return_value=self.act),
            mock.patch.object(q, "_grouped_stack_bytes", side_effect=lambda b, d, config=None: 1),
            mock.patch("auto_round.utils.device.probe_usable_bytes", return_value=16 * GB),
        ):
            q._maybe_auto_linear_loop_for_tuning(object(), [object()], 8, 20, ctx_cfg, model)
        self.assertEqual(self.cfg._experts_implementation, "linear_loop")
        self.assertEqual(ctx_cfg._experts_implementation, "linear_loop")

    def test_snapshot_parks_to_host_so_not_charged_as_guaranteed(self):
        # boundary between the old 10/14 and new 6/14 multipliers: this lane
        # must KEEP grouped (a working knife-edge lane must not be switched)
        q = self.q
        state = {"cuda:1": 20 * GB}  # remaining: 10/14 -> 14.3 (old switch), 6/14 -> 8.6
        act = {"cuda:1": 5 * GB}
        with (
            _envs_auto(),
            mock.patch.object(q, "_logical_state_by_device", return_value=state),
            mock.patch.object(q, "_activation_bytes_by_device", return_value=act),
            mock.patch.object(q, "_grouped_stack_bytes", side_effect=lambda b, d, config=None: 1),
            mock.patch("auto_round.utils.device.probe_usable_bytes", return_value=16 * GB),
        ):
            q._maybe_auto_linear_loop_for_tuning(object(), [object()], 8, 20, self.cfg, None)
        self.assertEqual(self.cfg._experts_implementation, "linear_grouped")

    def test_second_run_same_process_re_decides(self):
        # the done-flag is keyed by the run's config: a second model
        # quantized in the same process must get its own auto decision
        q = self.q
        q._MOE_IMPL_AUTO_LINEAR_LOOP_DONE = True  # a previous run finished
        cfg2 = _Cfg()
        with (
            _envs_auto(),
            mock.patch.object(q, "_logical_state_by_device", return_value=self.state),
            mock.patch.object(q, "_activation_bytes_by_device", return_value=self.act),
            mock.patch.object(q, "_grouped_stack_bytes", side_effect=lambda b, d, config=None: 1),
            mock.patch("auto_round.utils.device.probe_usable_bytes", return_value=16 * GB),
        ):
            q._maybe_auto_linear_loop_for_tuning(object(), [object()], 8, 20, cfg2, None)
        self.assertEqual(cfg2._experts_implementation, "linear_loop")  # decided, not skipped

    def test_dense_block_no_stacks_no_decision(self):
        # no grouped stacks homed: not the decision point, flag stays unset
        self._run(stacks=False)
        self.assertEqual(self.cfg._experts_implementation, "linear_grouped")
        self.assertFalse(self.q._MOE_IMPL_AUTO_LINEAR_LOOP_DONE)