Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

61.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Batched same-shape wrap-search tests: grouping, bit-parity, deferral protocol."""

import os
import threading
import types
import unittest
from unittest import mock

import torch

import auto_round.algorithms.quantization.search_dispatch as search_dispatch
from auto_round.algorithms.quantization.search_dispatch import run_batched_wrap_search


def _mk_inputs(seed=0, shape=(6, 128), dtype="int", bits=4, thresh=1e-5, device="cpu"):
    from auto_round.data_type.utils import resolve_optimized_init_scale_fn

    g = torch.Generator().manual_seed(seed)
    w = torch.randn(*shape, generator=g)
    im = torch.rand(shape[1], generator=g) + 0.5  # RAW per-column imatrix, as production stages it
    fn = resolve_optimized_init_scale_fn(dtype, thresh)
    return w, im, dtype, bits, thresh, device, fn


class FakeDeferred:
    """Duck-typed deferred wrapper: stages inputs, finalizes with the result."""

    supports_batched_search = True

    def __init__(self, seed=0, shape=(6, 128), dtype="int", bits=4, device="cpu", thresh=1e-5, search_fn=None):
        w, im, dt, b, th, dev, fn = _mk_inputs(seed, shape, dtype, bits, thresh, device)
        if search_fn is not None:
            fn = search_fn
        self._deferred_search_inputs = (w, dt, b, im, th, fn)
        self.init_scale = None
        self.finalized_on_thread = None
        self.name = f"m{seed}"
        self.orig_layer = types.SimpleNamespace(group_size=128)

    def _run_deferred_search_now(self):
        from auto_round.data_type.utils import reshape_imatrix_for_weight

        w, _dt, b, im_raw, _th, fn = self._deferred_search_inputs
        self.init_scale = fn(w, b, reshape_imatrix_for_weight(im_raw, w, self.orig_layer.group_size))
        self._deferred_search_inputs = None

    def finalize_batched_search(self, init_scale):
        self.init_scale = init_scale
        self._deferred_search_inputs = None
        self.finalized_on_thread = threading.current_thread().name


class TestNonStackableAndDictResults(unittest.TestCase):
    def test_split_leading_dicts(self):
        # GGUF K-quant funcs return scale/zp as dicts whose tensor values
        # carry the flattened batch dim; sharing the whole dict per module
        # gives every layer N-times the metadata (CI: gguf q2_k export)
        from auto_round.algorithms.quantization.rtn.batched_search import _split_leading

        scale = {"scale": torch.arange(8.0).reshape(8, 1), "d_scale": torch.arange(4.0).reshape(4, 1), "meta": None}
        parts = _split_leading(scale, 2)
        self.assertEqual(len(parts), 2)
        for i, part in enumerate(parts):
            self.assertEqual(part["scale"].shape, (4, 1))
            self.assertTrue(torch.equal(part["scale"], scale["scale"].reshape(2, 4, 1)[i]))
            self.assertEqual(part["d_scale"].shape, (2, 1))
            self.assertIsNone(part["meta"])  # non-tensor values are shared

    def test_search_fn_stackable_predicate(self):
        import functools

        from auto_round.algorithms.quantization.search_dispatch import _search_fn_stackable
        from auto_round.data_type.int import search_scales
        from auto_round.data_type.nvfp import search_nvfp4_scale

        self.assertFalse(_search_fn_stackable(search_nvfp4_scale))
        self.assertFalse(_search_fn_stackable(functools.partial(search_nvfp4_scale)))

        def shim(w, bits, im):
            return None

        shim._torchdynamo_orig_callable = search_nvfp4_scale
        self.assertFalse(_search_fn_stackable(shim))  # compiled wrappers unwrap
        self.assertTrue(_search_fn_stackable(search_scales))
        self.assertTrue(_search_fn_stackable(functools.partial(search_scales)))

    def test_nv_search_group_runs_per_module(self):
        # the nv-fp4 search is not batch-aware; its groups must never receive
        # a stacked weight (CI: test_nvfp4_alg_ext IndexError)
        import functools

        from auto_round.data_type.nvfp import search_nvfp4_scale

        calls = []

        def per_module(w, bits, im):
            calls.append(tuple(w.shape))
            return torch.ones(w.shape[0], 1)

        fn = functools.wraps(search_nvfp4_scale)(per_module)
        wrappers = [FakeDeferred(seed=1, search_fn=fn), FakeDeferred(seed=2, search_fn=fn)]
        self.assertTrue(run_batched_wrap_search(wrappers))
        self.assertEqual(calls, [(6, 128), (6, 128)])  # two single-module calls, never [2, 6, 128]
        for wr in wrappers:
            self.assertIsNotNone(wr.init_scale)


class TestBatchedBitParity(unittest.TestCase):
    def test_stacked_search_matches_individual_exactly(self):
        fakes = [FakeDeferred(seed=i) for i in range(5)]
        individual = [_individual(f) for f in fakes]
        stacked_w = torch.stack([f._deferred_search_inputs[0] for f in fakes])
        stacked_im = _stacked_im(fakes, stacked_w)
        batched = fakes[0]._deferred_search_inputs[5](stacked_w, 4, stacked_im)
        for i, ref in enumerate(individual):
            self.assertTrue(torch.equal(batched[i], ref), f"module {i} diverged")

    def test_parity_across_shapes_and_bits(self):
        for shape, bits in [((8, 128), 4), ((13, 128), 3), ((4, 64), 2)]:
            fakes = [FakeDeferred(seed=i + bits * 10, shape=shape, bits=bits) for i in range(4)]
            refs = [
                f._deferred_search_inputs[5](f._deferred_search_inputs[0], bits, f._deferred_search_inputs[3])
                for f in fakes
            ]
            _sw = torch.stack([f._deferred_search_inputs[0] for f in fakes])
            batched = fakes[0]._deferred_search_inputs[5](_sw, bits, _stacked_im(fakes, _sw))
            for i, ref in enumerate(refs):
                self.assertTrue(torch.equal(batched[i], ref))


class TestRunBatchedWrapSearch(unittest.TestCase):
    def test_batches_same_shape_and_finalizes(self):
        fakes = [FakeDeferred(seed=i) for i in range(4)] + [FakeDeferred(seed=99, shape=(3, 128))]
        expected = [_individual(f) for f in fakes]
        handled = run_batched_wrap_search(fakes)
        self.assertTrue(handled)
        for f, ref in zip(fakes, expected):
            self.assertIsNone(f._deferred_search_inputs)
            self.assertIsNotNone(f.init_scale)
            self.assertTrue(torch.equal(f.init_scale, ref), f.name)

    def test_single_deferred_runs_individually(self):
        fakes = [FakeDeferred(seed=0)]
        handled = run_batched_wrap_search(fakes)
        self.assertTrue(handled)  # still handled (individual fallback), inputs consumed
        self.assertIsNotNone(fakes[0].init_scale)

    def test_shape_mismatch_groups_separately(self):
        a = [FakeDeferred(seed=i, shape=(6, 128)) for i in range(2)]
        b = [FakeDeferred(seed=i, shape=(5, 128)) for i in range(2)]
        run_batched_wrap_search(a + b)
        for f in a + b:
            self.assertIsNotNone(f.init_scale)

    def test_dtype_mismatch_groups_separately(self):
        a = FakeDeferred(seed=0)
        b = FakeDeferred(seed=1, dtype="mx_fp4")
        run_batched_wrap_search([a, b])  # different resolved fn -> separate groups, no raise
        self.assertIsNotNone(a.init_scale)
        self.assertIsNotNone(b.init_scale)

    def test_distinct_search_fns_never_share_a_batch(self):
        def fake_fn_a(w, bits, im):
            return w.abs().amax(dim=-1).squeeze(-1)

        def fake_fn_b(w, bits, im):
            return -w.abs().amax(dim=-1).squeeze(-1)

        a = FakeDeferred(seed=0, search_fn=fake_fn_a)
        b = FakeDeferred(seed=1, search_fn=fake_fn_b)
        exp_a = fake_fn_a(a._deferred_search_inputs[0], 4, a._deferred_search_inputs[3])
        exp_b = fake_fn_b(b._deferred_search_inputs[0], 4, b._deferred_search_inputs[3])
        run_batched_wrap_search([a, b])
        # each kept its own search semantics (no shared stack across fns)
        self.assertTrue(torch.equal(a.init_scale, exp_a))
        self.assertTrue(torch.equal(b.init_scale, exp_b))

    def test_empty_list(self):
        self.assertFalse(run_batched_wrap_search([]))

    def test_batch_chunking_respects_cap(self):
        fakes = [FakeDeferred(seed=i) for i in range(6)]
        calls = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            calls.append(len(tensors))
            return real_stack(tensors, *a, **k)

        with mock.patch.object(torch, "stack", side_effect=spy_stack):
            run_batched_wrap_search(fakes, max_batch=2)
        # 3 chunks x (weights + imatrices) = 6 stacks, each of size 2
        self.assertEqual(sorted(calls), [2] * 6)
        for f in fakes:
            self.assertIsNotNone(f.init_scale)

    def test_element_budget_caps_batch_size(self):
        # the ELEMENT budget (not the VRAM probe) binds: budget mocked to two
        # modules' worth of elements -> 6 modules split into batches of 2
        fakes = [FakeDeferred(seed=i, shape=(64, 32)) for i in range(6)]
        calls = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            calls.append(len(tensors))
            return real_stack(tensors, *a, **k)

        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        per_module = 2 * 64 * 32  # weight + imatrix elements
        with mock.patch.object(torch, "stack", side_effect=spy_stack), mock.patch.object(
            dispatch_mod, "probe_usable_bytes", return_value=2**40
        ), mock.patch.object(dispatch_mod, "_wrap_batch_max_elems", return_value=2 * per_module):
            run_batched_wrap_search(fakes)
        self.assertEqual(calls, [2, 2, 2, 2, 2, 2])  # 3 batches x 2 stacks (weights + imatrix)

    def test_env_gb_override_caps_batches(self):
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        # tiny modules (0.26M elems each): 1 GiB budget would allow ~1000 -> force 2 per batch via GB override
        fakes = [FakeDeferred(seed=i, shape=(1024, 128)) for i in range(4)]
        calls = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            calls.append(len(tensors))
            return real_stack(tensors, *a, **k)

        with mock.patch.object(dispatch_mod.envs, "AR_SEARCH_BATCH_GB", 0.001), mock.patch.object(
            torch, "stack", side_effect=spy_stack
        ):
            run_batched_wrap_search(fakes)
        # 0.001 GiB = ~268K elements -> 268214 // (2*1024*128=262144) = 1 module/batch;
        # 4 batches x (weights + imatrices) = 8 stacks of size 1
        self.assertEqual(calls, [1] * 8)
        for f in fakes:
            self.assertIsNotNone(f.init_scale)

    def test_env_gb_override_invalid_raises(self):
        import os

        from auto_round import envs as envs_mod

        fakes = [FakeDeferred(seed=i) for i in range(4)]
        # through the real parser (module __getattr__ -> lambda raises)
        os.environ["AR_SEARCH_BATCH_GB"] = "not-a-number"
        try:
            with self.assertRaises(ValueError):
                run_batched_wrap_search(fakes)
        finally:
            os.environ.pop("AR_SEARCH_BATCH_GB", None)
            # sibling patch.object tests may leave a static module attr behind
            if "AR_SEARCH_BATCH_GB" in vars(envs_mod):
                delattr(envs_mod, "AR_SEARCH_BATCH_GB")

    def test_kill_switch_disables(self):
        fakes = [FakeDeferred(seed=i) for i in range(3)]
        with mock.patch.object(search_dispatch.envs, "AR_DISABLE_BATCHED_SEARCH", True):
            handled = run_batched_wrap_search(fakes)
        self.assertFalse(handled)
        self.assertIsNone(fakes[0].init_scale)  # inputs untouched: caller runs them per module
        for f in fakes:
            f._run_deferred_search_now()  # the caller's documented fallback
        self.assertIsNotNone(fakes[0].init_scale)


def _individual(fake):
    from auto_round.data_type.utils import reshape_imatrix_for_weight

    w, _dt, b, im_raw, _th, fn = fake._deferred_search_inputs
    return fn(w, b, reshape_imatrix_for_weight(im_raw, w, fake.orig_layer.group_size))


def _stacked_im(fakes, stacked_w):
    from auto_round.algorithms.quantization.search_dispatch import _materialize_wrap_imatrix

    return _materialize_wrap_imatrix(fakes, stacked_w)


class TestV2DeferralProtocol(unittest.TestCase):
    def _bare_v2(self):
        from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundOptimizedWrapperLinear

        w = object.__new__(SignRoundOptimizedWrapperLinear)
        w.init_scale = None
        w._deferred_search_inputs = None
        w.orig_layer = types.SimpleNamespace(group_size=128)
        return w

    def test_run_now_and_finalize_assign(self):
        from auto_round.data_type.utils import resolve_optimized_init_scale_fn

        w = self._bare_v2()
        weight = torch.randn(6, 128)
        imatrix_raw = torch.rand(128) + 0.5  # raw column, as production stages it
        fn = resolve_optimized_init_scale_fn("int", 1e-5)
        w._deferred_search_inputs = (weight, "int", 4, imatrix_raw, 1e-5, fn)
        from auto_round.data_type.utils import reshape_imatrix_for_weight

        ref = fn(weight, 4, reshape_imatrix_for_weight(imatrix_raw, weight, 128))
        w._run_deferred_search_now()
        self.assertTrue(torch.equal(w.init_scale, ref))
        self.assertIsNone(w._deferred_search_inputs)

        w2 = self._bare_v2()
        w2._deferred_search_inputs = (weight, "int", 4, imatrix_raw, 1e-5, fn)
        w2.finalize_batched_search(ref)
        self.assertTrue(torch.equal(w2.init_scale, ref))
        self.assertIsNone(w2._deferred_search_inputs)


class TestRealV2Construction(unittest.TestCase):
    """The kwarg must survive the REAL base __init__ chain (regression: kwargs swallowed it)."""

    def _layer(self):
        import torch.nn as nn

        layer = nn.Linear(128, 64, bias=False)
        layer.data_type = "int"
        layer.bits = 4
        layer.sym = True
        layer.group_size = 128
        layer.iters = 10
        layer.act_bits = 16
        return layer

    def _make(self, defer_search):
        from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundOptimizedWrapperLinear

        layer = self._layer()
        return SignRoundOptimizedWrapperLinear(
            layer,
            enable_minmax_tuning=False,
            enable_norm_bias_tuning=False,
            enable_torch_compile=False,
            device="cpu",
            defer_search=defer_search,
        )

    def test_deferred_stages_through_real_init(self):
        w = self._make(defer_search=True)
        self.assertIsNotNone(w._deferred_search_inputs, "defer_search was swallowed by the base __init__ kwargs")
        self.assertIsNone(w.init_scale)
        w._run_deferred_search_now()
        self.assertIsNotNone(w.init_scale)

    def test_non_deferred_searches_inline_through_real_init(self):
        w = self._make(defer_search=False)
        self.assertIsNone(w._deferred_search_inputs)
        self.assertIsNotNone(w.init_scale)

    def test_kwarg_absent_by_default(self):
        w = self._make(defer_search=False)
        self.assertIsNone(w._deferred_search_inputs)  # old callers unchanged


class TestCpuFallbackPreservesSearchSettings(unittest.TestCase):
    """The CPU fallback must only change WHERE the search runs, not WHICH search."""

    def test_rebuild_carries_opt_rtn_and_neuqi_flags(self):
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn import batched_search

        layer = nn.Linear(8, 8, bias=False)
        layer.data_type = "int"
        layer.bits = 4
        layer.sym = True
        layer.group_size = -1
        layer.iters = 0
        layer.act_bits = 16

        class _Failed:
            orig_layer = layer
            disable_opt_rtn = False  # optimized RTN active on the failed wrapper
            enable_neuqi = True  # NeuQI active on the failed wrapper

            def unwrapper(self, cfg):
                raise torch.OutOfMemoryError("simulated per-module OOM")

        recorded = {}

        class _Recorder:
            def __call__(self, lyr, **kw):
                recorded.update(kw)
                recorded["_positional_is_layer"] = lyr is layer

                class _Out:
                    def unwrapper(self, cfg):
                        return "cpu_result"

                return _Out()

        with mock.patch.object(batched_search, "WrapperLinear", _Recorder()), mock.patch.object(
            batched_search, "dump_oom_tensor_census_", lambda *a, **k: None
        ):
            out = batched_search._unwrap_with_cpu_fallback(_Failed(), "model.l0")

        self.assertEqual(out, "cpu_result")
        self.assertTrue(recorded.pop("_positional_is_layer"))
        self.assertFalse(recorded["disable_opt_rtn"])  # stays OFF, not the True default
        self.assertTrue(recorded["enable_neuqi"])  # stays ON, not the False default


class TestStackableLayoutGate(unittest.TestCase):
    """Layouts whose quant math is not row-stackable must finish per-module."""

    def setUp(self):
        self._env = mock.patch.dict(os.environ, {"AR_DISABLE_CROSS_DEVICE_SEARCH": "1"})
        self._env.start()
        self.addCleanup(self._env.stop)

    def _layer(self, seed, bits, group_size, act_bits=16):
        import torch.nn as nn

        g = torch.Generator().manual_seed(seed)
        layer = nn.Linear(128, 64, bias=False)
        with torch.no_grad():
            layer.weight.copy_(torch.randn(64, 128, generator=g))
        layer.data_type = "int"
        layer.bits = bits
        layer.sym = True
        layer.group_size = group_size
        layer.iters = 0
        layer.act_bits = act_bits
        if act_bits <= 8:
            layer.act_data_type = "int"
            layer.act_scale_dtype = torch.float16
            layer.act_sym = False
            layer.act_group_size = 128
            layer.act_dynamic = False
        layer.scale_dtype = torch.float16
        return layer

    def _wrapper(self, layer):
        from auto_round.wrapper import WrapperLinear

        return WrapperLinear(
            layer,
            device="cpu",
            enable_minmax_tuning=False,
            enable_norm_bias_tuning=False,
            enable_round_tuning=False,
            enable_torch_compile=False,
            disable_opt_rtn=True,
            iters=0,
        )

    def test_stackable_layout_matrix(self):
        from auto_round.algorithms.quantization.rtn.batched_search import _stackable_layout

        self.assertTrue(_stackable_layout(self._wrapper(self._layer(0, 4, 128))))  # row groups
        self.assertTrue(_stackable_layout(self._wrapper(self._layer(1, 4, -1))))  # per-channel
        self.assertFalse(_stackable_layout(self._wrapper(self._layer(2, 4, 0))))  # per-tensor scale
        self.assertFalse(_stackable_layout(self._wrapper(self._layer(3, 8, (128, 128)))))  # 2-D blocks
        self.assertFalse(_stackable_layout(self._wrapper(self._layer(4, 16, 128, act_bits=8))))  # act-only

    def _parity(self, bits, group_size, act_bits=16):
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        with torch.no_grad():
            serial = []
            for i in range(3):
                layer = self._layer(seed=i, bits=bits, group_size=group_size, act_bits=act_bits)
                out = self._wrapper(layer).unwrapper({})
                serial.append(out.weight.data.clone())

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = self._layer(seed=i, bits=bits, group_size=group_size, act_bits=act_bits)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", self._wrapper(layer)))
        run_batched_rtn_search(model, staged)
        for i in range(3):
            self.assertTrue(
                torch.equal(getattr(model, f"l{i}").weight.data, serial[i]),
                f"module {i} diverged from the serial path",
            )

    def test_per_tensor_group_size_matches_serial(self):
        self._parity(bits=4, group_size=0)

    def test_act_only_layer_matches_serial(self):
        self._parity(bits=16, group_size=128, act_bits=8)


class TestWorkerBucketKeys(unittest.TestCase):
    def test_multi_worker_keys_are_device_parseable(self):
        """Regression: the threading key once carried the chunk entry dict itself."""
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn import batched_search

        model = nn.Module()
        staged = []
        for i in range(2):
            layer = TestBatchedRtnSearchParity()._layer(seed=i)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", TestBatchedRtnSearchParity()._make_wrapper(layer)))
        captured = {}

        def fake_run(keyed, fn):
            captured["keys"] = list(keyed.keys())

        import auto_round.algorithms.quantization.search_dispatch as _sd

        with mock.patch.object(_sd, "pick_search_worker_devices", return_value=["cpu", "cuda:9"]), mock.patch.object(
            _sd, "probe_usable_bytes", return_value=2**40
        ), mock.patch.object(batched_search, "run_items_by_device", side_effect=fake_run):
            # max_batch=1 -> two singleton chunks spread round-robin over the
            # two candidates -> two distinct worker buckets; run_items_by_device
            # is mocked so nothing actually moves to cuda:9
            batched_search.run_batched_rtn_search(model, staged, max_batch=1)
        # distinct worker bucket vs home -> multi-bucket path taken; every key must parse
        self.assertEqual(sorted(captured["keys"]), ["cpu", "cuda:9"])
        for k in captured["keys"]:
            torch.device(k)  # the exact operation that crashed on the server


class TestEagerUnwrap(unittest.TestCase):
    def test_compiled_fn_unwraps_to_original(self):
        import torch

        def orig(x):
            return x * 2

        compiled = torch.compile(orig)
        unwrapped = getattr(compiled, "_torchdynamo_orig_callable", None) or compiled
        self.assertIs(unwrapped, orig)  # offloaded calls bypass the compiled wrapper
        self.assertEqual(unwrapped(torch.ones(2)).sum().item(), 4.0)

    def test_plain_fn_passes_through(self):
        fn = lambda x: x  # noqa: E731
        self.assertIs(getattr(fn, "_torchdynamo_orig_callable", None) or fn, fn)


class TestKwargRelocation(unittest.TestCase):
    def test_non_tensors_and_cpu_scalars_untouched(self):
        from auto_round.algorithms.quantization.rtn.batched_search import _relocate_tensor_kwargs

        scal = torch.tensor(1.0)
        kwargs = {"bits": 4, "v": scal, "group_size": 128, "data_type": "int"}
        out = _relocate_tensor_kwargs(dict(kwargs), "cpu")
        self.assertIs(out["v"], scal)  # cpu scalar: same object, no copy
        self.assertEqual(out["bits"], 4)

    def test_returns_same_dict_semantics(self):
        from auto_round.algorithms.quantization.rtn.batched_search import _relocate_tensor_kwargs

        kwargs = {"imatrix": None, "tensor_min": None}
        out = _relocate_tensor_kwargs(kwargs, "cpu")
        self.assertIsNone(out["imatrix"])


class TestSearchWorkerPicking(unittest.TestCase):
    def _pick(self, working_set, free_map, home="cuda:0"):
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        with mock.patch.object(torch.cuda, "device_count", return_value=len(free_map)), mock.patch.object(
            dispatch_mod, "probe_usable_bytes", side_effect=lambda k: free_map.get(k)
        ):
            return dispatch_mod.pick_search_worker_devices(working_set, home_device=home)

    def test_full_home_goes_to_idle_devices(self):
        free = {"cuda:0": 1 * 2**30, "cuda:1": 12 * 2**30, "cuda:2": 12 * 2**30}
        ws = 4 * 2**30
        self.assertEqual(self._pick(ws, free), ["cuda:1", "cuda:2"])  # home excluded, idles viable

    def test_all_full_falls_back_to_home(self):
        free = {"cuda:0": 0, "cuda:1": 1 * 2**30}
        self.assertEqual(self._pick(4 * 2**30, free), ["cuda:0"])

    def test_home_participates_when_it_fits(self):
        free = {"cuda:0": 10 * 2**30, "cuda:1": 10 * 2**30}
        self.assertEqual(self._pick(2 * 2**30, free), ["cuda:0", "cuda:1"])

    def test_kill_switch_pins_home(self):
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        with mock.patch.object(dispatch_mod.envs, "AR_DISABLE_CROSS_DEVICE_SEARCH", True):
            self.assertEqual(dispatch_mod.pick_search_worker_devices(4 * 2**30, home_device="cuda:3"), ["cuda:3"])

    def test_no_cuda_returns_home(self):
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        with mock.patch.object(torch.cuda, "device_count", return_value=0):
            self.assertEqual(dispatch_mod.pick_search_worker_devices(4 * 2**30, home_device="cpu"), ["cpu"])

    def test_hpu_lazy_stays_home_despite_idle_devices(self):
        # Habana lazy tensors execute on per-thread device contexts: a stacked
        # search spread onto worker threads reaches the write-back's host copy
        # without storage attached (CI: data_ptr()-> "no storage tensor
        # attached to lazy tensor"). Lazy mode must collapse to the home device.
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        class _Fleet:
            @staticmethod
            def device_count():
                return 4

        with mock.patch.object(dispatch_mod, "is_hpu_lazy_mode", return_value=True), mock.patch(
            "auto_round.utils.device_manager.get_ar_device", return_value=_Fleet()
        ), mock.patch.object(dispatch_mod, "probe_usable_bytes", side_effect=lambda k: 12 * 2**30):
            self.assertEqual(dispatch_mod.pick_search_worker_devices(2**30, home_device="hpu:0"), ["hpu:0"])

    def test_hpu_compile_mode_spreads_like_other_families(self):
        # compile mode runs the eager pipeline: concrete tensors survive worker
        # threads and cross-device moves, so spreading stays enabled there.
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod

        class _Fleet:
            @staticmethod
            def device_count():
                return 2

        with mock.patch.object(dispatch_mod, "is_hpu_lazy_mode", return_value=False), mock.patch(
            "auto_round.utils.device_manager.get_ar_device", return_value=_Fleet()
        ), mock.patch.object(dispatch_mod, "probe_usable_bytes", side_effect=lambda k: 12 * 2**30):
            self.assertEqual(dispatch_mod.pick_search_worker_devices(2**30, home_device="hpu:0"), ["hpu:0", "hpu:1"])

    def test_rtn_driver_offloads_to_viable_worker(self):
        # CPU-only: workers = [home] so behavior is the parity path already covered;
        # this pins that the bucket machinery runs end-to-end with the picker patched.
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        model = nn.Module()
        staged = []
        for i in range(2):
            layer = TestBatchedRtnSearchParity()._layer(seed=i)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", TestBatchedRtnSearchParity()._make_wrapper(layer)))
        with mock.patch(
            "auto_round.algorithms.quantization.search_dispatch.pick_search_worker_devices",
            return_value=["cpu"],
        ):
            run_batched_rtn_search(model, staged)
        for i in range(2):
            self.assertIsNotNone(getattr(model, f"l{i}").scale)


class TestWrapperBlockDrivesBatching(unittest.TestCase):
    def _fake_block(self):
        block = torch.nn.Module()
        for i in range(3):
            m = torch.nn.Linear(8, 8, bias=False)
            m.bits = 4
            setattr(block, f"l{i}", m)
        return block

    def test_protocol_class_deferred_and_batched(self):
        import auto_round.wrapper as wrapper_mod

        calls = {"created": 0, "finalized": 0}

        class FakeBatched(torch.nn.Module):
            supports_batched_search = True

            def __init__(self, layer, defer_search=False, **kwargs):
                super().__init__()
                self.orig_layer = layer
                self.init_scale = None
                self._deferred_search_inputs = None
                calls["created"] += 1
                assert defer_search is True
                w = layer.weight.data.reshape(-1, layer.weight.shape[1])
                from auto_round.data_type.utils import resolve_optimized_init_scale_fn

                self._deferred_search_inputs = (
                    w,
                    "int",
                    4,
                    None,  # uniform importance: production stages None, not ones
                    1e-5,
                    resolve_optimized_init_scale_fn("int", 1e-5),
                )

            def _run_deferred_search_now(self):
                w, _dt, b, im, _th, fn = self._deferred_search_inputs
                self.init_scale = fn(w, b, torch.ones_like(w) if im is None else im)
                self._deferred_search_inputs = None

            def finalize_batched_search(self, init_scale):
                self.init_scale = init_scale
                self._deferred_search_inputs = None
                calls["finalized"] += 1

        block = self._fake_block()
        q, u = wrapper_mod.wrapper_block(
            block, False, False, enable_torch_compile=False, device="cpu", wrapper_cls=FakeBatched
        )
        self.assertEqual(q, ["l0", "l1", "l2"])
        self.assertEqual(calls["created"], 3)
        self.assertEqual(calls["finalized"], 3)  # all three searched via the batch driver
        for m in (block.l0, block.l1, block.l2):
            self.assertIsNotNone(m.init_scale)

    def test_kill_switch_runs_per_module(self):
        import auto_round.algorithms.quantization.search_dispatch as dispatch_mod
        import auto_round.wrapper as wrapper_mod

        stats = {"inline": 0, "now": 0}

        class FakeBatched(torch.nn.Module):
            supports_batched_search = True

            def __init__(self, layer, defer_search=False, **kwargs):
                super().__init__()
                self.orig_layer = layer
                self.init_scale = None
                self._deferred_search_inputs = None
                if defer_search:
                    self._deferred_search_inputs = ("staged",)
                else:  # kill switch: the class must search inline, as the real wrapper does
                    stats["inline"] += 1
                    self.init_scale = torch.zeros(1)

            def _run_deferred_search_now(self):
                stats["now"] += 1
                self.init_scale = torch.zeros(1)
                self._deferred_search_inputs = None

            def finalize_batched_search(self, init_scale):
                raise AssertionError("must not batch under the kill switch")

        block = self._fake_block()
        with mock.patch.object(dispatch_mod.envs, "AR_DISABLE_BATCHED_SEARCH", True):
            wrapper_mod.wrapper_block(
                block, False, False, enable_torch_compile=False, device="cpu", wrapper_cls=FakeBatched
            )
        self.assertEqual(stats["inline"], 3)  # no deferral at all under the kill switch
        self.assertEqual(stats["now"], 0)
        self.assertIsNotNone(block.l0.init_scale)


class TestBatchedRtnSearchParity(unittest.TestCase):
    """iters=0 batching: stacked vs serial must be bit-identical (incl. the imatrix path)."""

    def setUp(self):
        # parity is asserted against a serial CPU arm; on GPU-rich boxes the
        # multigpu offload would move the batched arm to idle CUDA workers
        # (different device numerics) -- out of scope for these tests
        self._env = mock.patch.dict(os.environ, {"AR_DISABLE_CROSS_DEVICE_SEARCH": "1"})
        self._env.start()
        self.addCleanup(self._env.stop)

    def _layer(self, seed, sym=True, with_imatrix=True):
        import torch.nn as nn

        g = torch.Generator().manual_seed(seed)
        layer = nn.Linear(128, 64, bias=False)
        with torch.no_grad():
            layer.weight.copy_(torch.randn(64, 128, generator=g))
        layer.data_type = "int"
        layer.bits = 4
        layer.sym = sym
        layer.group_size = 128
        layer.iters = 0
        layer.act_bits = 16
        layer.scale_dtype = torch.float16
        if with_imatrix:
            layer.imatrix = torch.rand(128, generator=g) + 0.5
        return layer

    def _make_wrapper(self, layer):
        from auto_round.wrapper import WrapperLinear

        return WrapperLinear(
            layer,
            device="cpu",
            enable_minmax_tuning=False,
            enable_norm_bias_tuning=False,
            enable_round_tuning=False,
            enable_torch_compile=False,
            disable_opt_rtn=False,
            iters=0,
        )

    def _run(self, sym, with_imatrix):
        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        # serial arm (production callers run under no_grad)
        serial = []
        with torch.no_grad():
            for i in range(3):
                layer = self._layer(seed=i, sym=sym, with_imatrix=with_imatrix)
                w = self._make_wrapper(layer)
                out = w.unwrapper({})
                serial.append((out.weight.data.clone(), out.scale, out.zp))
        # batched arm
        import torch.nn as nn

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = self._layer(seed=i, sym=sym, with_imatrix=with_imatrix)
            setattr(model, f"l{i}", layer)
            w = self._make_wrapper(layer)
            staged.append((f"l{i}", w))
        run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"l{i}")
            ref_w, ref_scale, ref_zp = serial[i]
            self.assertTrue(torch.equal(got.weight.data, ref_w), f"weight mismatch module {i}")
            if isinstance(ref_scale, torch.Tensor):
                self.assertTrue(torch.equal(got.scale, ref_scale), f"scale mismatch module {i}")
            if ref_zp is not None and isinstance(ref_zp, torch.Tensor):
                self.assertTrue(torch.equal(got.zp, ref_zp), f"zp mismatch module {i}")

    def test_parity_sym_with_imatrix(self):
        self._run(sym=True, with_imatrix=True)

    def test_materialization_oom_falls_back_per_module(self):
        """An OOM while moving/copying the batch (before the quant call) must
        take the same per-module ladder as a stacked-call OOM, not abort."""
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn import batched_search
        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        serial = []
        with torch.no_grad():
            for i in range(3):
                layer = self._layer(seed=i, sym=True, with_imatrix=True)
                out = self._make_wrapper(layer).unwrapper({})
                serial.append((out.weight.data.clone(), out.scale, getattr(out, "zp", None)))

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = self._layer(seed=i, sym=True, with_imatrix=True)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", self._make_wrapper(layer)))

        with mock.patch.object(
            batched_search,
            "_expand_imatrix",
            side_effect=torch.OutOfMemoryError("simulated materialization OOM"),
        ):
            run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"l{i}")
            ref_w, ref_scale, ref_zp = serial[i]
            self.assertTrue(torch.equal(got.weight.data, ref_w), f"weight mismatch module {i}")
            self.assertTrue(torch.equal(got.scale, ref_scale), f"scale mismatch module {i}")

    def test_stacked_oom_falls_back_per_module(self):
        """A stacked-call OOM finishes the chunk per-module: every module ends
        up exactly as the serial arm produced it."""
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        # serial reference arm (real quant fns, untouched)
        serial = []
        with torch.no_grad():
            for i in range(3):
                layer = self._layer(seed=i, sym=True, with_imatrix=True)
                out = self._make_wrapper(layer).unwrapper({})
                serial.append((out.weight.data.clone(), out.scale, getattr(out, "zp", None)))

        class _OomOnStacked:
            # raises only on the stacked 3D call; the per-module fallback's 2D
            # calls delegate to the real quant fn. One shared instance: the
            # staged key derives from the callable, so the fake must key
            # identically across wrappers to form one chunk; the name lands
            # outside the non-stackable denylist
            def __init__(self, real):
                self.real = real
                self.__name__ = "quant_tensor"

            def __call__(self, w, *a, **kw):
                if w.dim() == 3:
                    raise torch.OutOfMemoryError("simulated stacked OOM")
                return self.real(w, *a, **kw)

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = self._layer(seed=i, sym=True, with_imatrix=True)
            setattr(model, f"l{i}", layer)
            w = self._make_wrapper(layer)
            staged.append((f"l{i}", w))
        fake = _OomOnStacked(staged[0][1].weight_quant_func)
        for _name, w in staged:
            w.weight_quant_func = fake
        run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"l{i}")
            ref_w, ref_scale, ref_zp = serial[i]
            self.assertTrue(torch.equal(got.weight.data, ref_w), f"weight mismatch module {i}")
            self.assertTrue(torch.equal(got.scale, ref_scale), f"scale mismatch module {i}")
            if isinstance(ref_zp, torch.Tensor):
                self.assertTrue(torch.equal(got.zp, ref_zp), f"zp mismatch module {i}")

    def test_parity_sym_no_imatrix(self):
        self._run(sym=True, with_imatrix=False)

    def test_parity_asym_with_imatrix(self):
        self._run(sym=False, with_imatrix=True)

    def test_parity_act_quant_runs_full_unwrapper_tail(self):
        """Regression (review R4-3): the stacked write-back used _apply_qdq +
        attach(orig_layer), skipping the unwrapper tail -- act-quantized layers
        (act_bits <= 8, W4A8/W4A4) silently lost WrapperWALayer and the act
        metadata the serial and singleton paths attach."""
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search
        from auto_round.wrapper import WrapperWALayer

        def _act_layer(seed):
            layer = self._layer(seed)
            layer.act_bits = 8  # enables the act tail in unwrapper
            layer.act_data_type = "int"
            layer.act_sym = False
            layer.act_dynamic = False  # static act: the tail computes act_scale
            layer.act_group_size = -1
            return layer

        # serial arm: the full tail attaches a WrapperWALayer
        with torch.no_grad():
            w = self._make_wrapper(_act_layer(0))
            serial_out = w.unwrapper({})
        self.assertIsInstance(serial_out, WrapperWALayer)

        # batched arm: same module type + act metadata + bit-identical weights
        model = nn.Module()
        staged = []
        for i in range(3):
            layer = _act_layer(seed=i)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", self._make_wrapper(layer)))
        run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"l{i}")
            self.assertIsInstance(got, WrapperWALayer, f"module {i} lost the act wrapper")
            self.assertIsNotNone(getattr(got, "act_quant_func", None), f"module {i} missing act_quant_func")
            # the tail sets static-act attrs on the wrapped orig layer
            self.assertTrue(hasattr(got.orig_layer, "act_scale"), f"module {i} missing act_scale")
            self.assertIsNotNone(getattr(got.orig_layer, "scale", None), f"module {i} missing weight scale")
        ref_w = None
        with torch.no_grad():
            w_ref = self._make_wrapper(_act_layer(0))
            ref_w = w_ref.unwrapper({}).weight.data.clone()
        self.assertTrue(torch.equal(model.l0.weight.data, ref_w))

    def _run_shape_dtype(self, out_f, in_f, gs, dtype):
        """Batched-vs-serial parity for arbitrary shapes/dtypes with an imatrix
        (regressions R6-2/R6-3: bf16 weights + fp32 imatrix; in % gs != 0)."""
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        def _layer(seed):
            import torch.nn as nn

            g = torch.Generator().manual_seed(seed)
            layer = nn.Linear(in_f, out_f, bias=False)
            with torch.no_grad():
                layer.weight.copy_(torch.randn(out_f, in_f, generator=g).to(dtype))
            layer.data_type = "int"
            layer.bits = 4
            layer.sym = False
            layer.group_size = gs
            layer.iters = 0
            layer.act_bits = 16
            layer.scale_dtype = torch.float32
            layer.imatrix = (torch.rand(in_f, generator=g) + 0.5).float()  # fp32, like the hook
            return layer

        with torch.no_grad():
            serial = []
            for i in range(3):
                w = self._make_wrapper(_layer(i))
                out = w.unwrapper({})
                serial.append((out.weight.data.clone(), out.scale))

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = _layer(i)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", self._make_wrapper(layer)))
        run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"l{i}")
            ref_w, ref_scale = serial[i]
            self.assertTrue(torch.equal(got.weight.data, ref_w), f"weight mismatch module {i}")
            self.assertTrue(torch.equal(got.scale, ref_scale), f"scale mismatch module {i}")

    def test_parity_bf16_weights_fp32_imatrix(self):
        self._run_shape_dtype(64, 128, 128, torch.bfloat16)

    def test_parity_fp16_weights_fp32_imatrix(self):
        self._run_shape_dtype(64, 128, 128, torch.float16)

    def test_parity_group_size_not_dividing_in_features(self):
        self._run_shape_dtype(3, 10, 4, torch.float32)

    def test_staged_key_normalizes_compiled_quant_fn(self):
        """Regression (R6-4): with enable_torch_compile, every wrapper carries a
        UNIQUE torch.compile wrapper around the SHARED eager fn; keying the raw
        callable collapsed all groups to singletons (batching silently dead)."""
        from auto_round.algorithms.quantization.rtn.batched_search import _staged_key

        class _Compiled:
            def __init__(self, orig):
                self._torchdynamo_orig_callable = orig

            def __call__(self, *a, **k):  # pragma: no cover
                raise AssertionError("compiled fn must not run here")

        def _shared_eager(w, bits, imatrix):  # pragma: no cover - identity stub
            return None

        layers, wrappers = [], []
        for i in range(2):
            layer = self._layer(seed=i)
            layers.append(layer)
            w = self._make_wrapper(layer)
            w.weight_quant_func = _Compiled(_shared_eager)  # distinct wrappers, shared orig
            wrappers.append(w)
        k0 = _staged_key(wrappers[0], wrappers[0].orig_layer.weight, None)
        k1 = _staged_key(wrappers[1], wrappers[1].orig_layer.weight, None)
        self.assertEqual(k0, k1)

    def test_quant_call_kwargs_never_relocates_imatrix_to_meta(self):
        """Regression (R6-1): a meta-resident stored weight must not become the
        imatrix relocation target."""
        layer = self._layer(seed=0)
        layer.imatrix = torch.rand(layer.weight.shape[1]) + 0.5
        w = self._make_wrapper(layer)
        # simulate the low-memory lane: stored weight on meta, real data via get_weight
        real = layer.weight.data.clone()
        layer.weight = torch.nn.Parameter(torch.empty_like(real, device="meta"), requires_grad=False)
        layer.get_weight = lambda: real
        kwargs = w._quant_call_kwargs(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
        im = kwargs.get("imatrix")
        self.assertIsNotNone(im)
        self.assertNotEqual(im.device.type, "meta")
        self.assertEqual(im.device.type, "cpu")  # wrapper device in these tests

    def test_threaded_write_back_swaps_compiled_act_fn_to_eager(self):
        """Regression (review R5-1): on worker threads the unwrapper act tail
        would make its FIRST call to the torch.compile-wrapped act_quant_func
        -- the same dynamo trace-lock race the weight fn had. The threaded
        write-back must swap BOTH compiled callables (weight AND act) to their
        eager originals and restore them afterwards."""
        from auto_round.algorithms.quantization.rtn.batched_search import swap_wrapper_callables_to_eager

        ran_eager = []

        def _fake_eager_act(x, **kwargs):
            ran_eager.append(True)
            return x, torch.ones(1), None

        class _Compiled:
            def __init__(self, orig):
                self._torchdynamo_orig_callable = orig

            def __call__(self, *a, **k):  # pragma: no cover - must never run on workers
                raise AssertionError("compiled callable ran while swapped out")

        w = self._make_wrapper(self._layer(0))
        compiled_w = _Compiled(lambda *a, **k: (None, None, None))
        compiled_a = _Compiled(_fake_eager_act)
        w.weight_quant_func = compiled_w
        w.act_quant_func = compiled_a

        restore = swap_wrapper_callables_to_eager(w)
        self.assertIsNotNone(restore)
        # swapped: both attrs now point at the eager originals
        self.assertIs(w.weight_quant_func, compiled_w._torchdynamo_orig_callable)
        self.assertIs(w.act_quant_func, _fake_eager_act)
        w.act_quant_func(None)  # the eager act callable is what runs on the worker
        restore()
        # restored: both compiled callables back in place
        self.assertIs(w.weight_quant_func, compiled_w)
        self.assertIs(w.act_quant_func, compiled_a)

        # wrappers without compiled callables need no swap
        w2 = self._make_wrapper(self._layer(1))
        self.assertIsNone(swap_wrapper_callables_to_eager(w2))

    def _conv1d_layer(self, seed, nf, nx):
        from transformers.pytorch_utils import Conv1D

        g = torch.Generator().manual_seed(seed)
        layer = Conv1D(nf, nx)  # weight is [nx, nf]; quant math runs on the transpose
        with torch.no_grad():
            layer.weight.copy_(torch.randn(nx, nf, generator=g))
        layer.data_type = "int"
        layer.bits = 4
        layer.sym = True
        layer.group_size = min(nx, 128)
        layer.iters = 0
        layer.act_bits = 16
        layer.scale_dtype = torch.float16
        layer.imatrix = torch.rand(nx, generator=g) + 0.5
        return layer

    def _run_conv1d(self, nf, nx):
        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        with torch.no_grad():
            serial = []
            for i in range(3):
                w = self._make_wrapper(self._conv1d_layer(seed=i, nf=nf, nx=nx))
                out = w.unwrapper({})
                serial.append((out.weight.data.clone(), out.scale, out.zp))
        import torch.nn as nn

        model = nn.Module()
        staged = []
        for i in range(3):
            layer = self._conv1d_layer(seed=i, nf=nf, nx=nx)
            setattr(model, f"c{i}", layer)
            staged.append((f"c{i}", self._make_wrapper(layer)))
        run_batched_rtn_search(model, staged)
        for i in range(3):
            got = getattr(model, f"c{i}")
            ref_w, ref_scale, ref_zp = serial[i]
            self.assertTrue(torch.equal(got.weight.data, ref_w), f"weight mismatch module {i}")
            self.assertEqual(tuple(got.weight.data.shape), (nx, nf))  # Conv1D layout restored
            if isinstance(ref_scale, torch.Tensor):
                self.assertTrue(torch.equal(got.scale, ref_scale), f"scale mismatch module {i}")

    def _run_kwargs_mix(self, mutate):
        # two same-shape modules whose per-layer quant kwargs differ must NOT
        # share a stacked chunk (chunk[0]'s kwargs would silently apply to
        # both), and each must still match its serial result bit-for-bit
        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        with torch.no_grad():
            serial = []
            for i in range(2):
                layer = self._layer(seed=i, sym=True, with_imatrix=False)
                mutate(layer, i)
                w = self._make_wrapper(layer)
                out = w.unwrapper({})
                serial.append(out.weight.data.clone())
        import torch.nn as nn

        model = nn.Module()
        staged = []
        for i in range(2):
            layer = self._layer(seed=i, sym=True, with_imatrix=False)
            mutate(layer, i)
            setattr(model, f"k{i}", layer)
            staged.append((f"k{i}", self._make_wrapper(layer)))
        stacks = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            stacks.append(len(tensors))
            return real_stack(tensors, *a, **k)

        with mock.patch.object(torch, "stack", side_effect=spy_stack):
            run_batched_rtn_search(model, staged)
        # differing kwargs -> per-module: every stacked call carries exactly
        # ONE row (1-row chunks ride the stacked machinery for worker
        # placement; nothing is ever SHARED across the differing modules)
        self.assertTrue(stacks, "expected per-module stacked calls")
        self.assertTrue(all(s == 1 for s in stacks), f"shared batch: {stacks}")
        for i in range(2):
            got = getattr(model, f"k{i}")
            self.assertTrue(torch.equal(got.weight.data, serial[i]), f"weight mismatch module {i}")

    def test_mixed_scale_dtype_never_shares_batch(self):
        def mutate(layer, i):
            layer.scale_dtype = torch.float16 if i == 0 else torch.bfloat16

        self._run_kwargs_mix(mutate)

    def test_super_bits_on_one_layer_never_shares_batch(self):
        def mutate(layer, i):
            if i == 1:
                layer.super_bits = 2
                layer.super_group_size = 16

        self._run_kwargs_mix(mutate)

    def test_global_scale_difference_never_shares_batch(self):
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn.batched_search import run_batched_rtn_search

        with torch.no_grad():
            serial = []
            for i in range(2):
                layer = self._layer(seed=i, sym=True, with_imatrix=False)
                w = self._make_wrapper(layer)
                w.weight_global_scale = torch.tensor(1.0 + i)
                out = w.unwrapper({})
                serial.append(out.weight.data.clone())

        model = nn.Module()
        staged = []
        for i in range(2):
            layer = self._layer(seed=i, sym=True, with_imatrix=False)
            w = self._make_wrapper(layer)
            w.weight_global_scale = torch.tensor(1.0 + i)
            setattr(model, f"g{i}", layer)
            staged.append((f"g{i}", w))
        stacks = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            stacks.append(len(tensors))
            return real_stack(tensors, *a, **k)

        with mock.patch.object(torch, "stack", side_effect=spy_stack):
            run_batched_rtn_search(model, staged)
        # differing kwargs -> per-module: every stacked call carries exactly
        # ONE row (1-row chunks ride the stacked machinery for worker
        # placement; nothing is ever SHARED across the differing modules)
        self.assertTrue(stacks, "expected per-module stacked calls")
        self.assertTrue(all(s == 1 for s in stacks), f"shared batch: {stacks}")
        for i in range(2):
            got = getattr(model, f"g{i}")
            self.assertTrue(torch.equal(got.weight.data, serial[i]), f"weight mismatch module {i}")

    def test_parity_conv1d_square(self):
        self._run_conv1d(nf=64, nx=64)

    def test_parity_conv1d_nonsquare(self):
        # non-square is where the missing transpose is a hard crash, not silent corruption
        self._run_conv1d(nf=64, nx=128)

    def test_oom_chunk_falls_back_per_module(self):
        import torch.nn as nn

        from auto_round.algorithms.quantization.rtn import batched_search

        model = nn.Module()
        staged = []
        for i in range(2):
            layer = self._layer(seed=i)
            setattr(model, f"l{i}", layer)
            staged.append((f"l{i}", self._make_wrapper(layer)))
        real_fn = staged[0][1].weight_quant_func
        calls = {"n": 0, "raised": 0}

        def boom(*a, **k):
            calls["n"] += 1
            if calls["n"] == 1:  # only the stacked call raises; per-module fallback delegates
                calls["raised"] += 1
                raise torch.OutOfMemoryError("simulated")
            return real_fn(*a, **k)

        # patch BOTH wrappers with the SAME callable so the group key still matches
        with mock.patch.object(staged[0][1], "weight_quant_func", boom), mock.patch.object(
            staged[1][1], "weight_quant_func", boom
        ):
            batched_search.run_batched_rtn_search(model, staged)
        self.assertEqual(calls["raised"], 1)  # the stacked call raised exactly once
        self.assertEqual(calls["n"], 3)  # then the two per-module fallbacks delegated to the real fn
        for i in range(2):
            got = getattr(model, f"l{i}")
            self.assertIsNotNone(getattr(got, "scale", None))  # per-module fallback quantized it


class FakeAnchor:
    """Duck-typed deferred NeuUQI anchor: stages inputs, applies searched margins."""

    def __init__(self, seed, shape=(8, 128), sym=True, with_imatrix=True):
        import types as _types

        g = torch.Generator().manual_seed(seed)
        w = torch.randn(*shape, generator=g)
        im = (torch.rand(shape[1], generator=g) + 0.5) if with_imatrix else None
        self._deferred_neuqi_anchor = (w, 4, shape[1], im, sym)
        self._applied = None
        self.device = "cpu"
        self.orig_layer = _types.SimpleNamespace(bits=4, group_size=shape[1])
        self.weight_min = torch.zeros(w.numel() // shape[1])
        self.weight_max = torch.zeros(w.numel() // shape[1])

    def _run_deferred_neuqi_anchor_now(self):
        from auto_round.data_type.neuqi import neuqi_anchor_margins

        w, bits, gs, im, sym = self._deferred_neuqi_anchor
        wmin, wmax = neuqi_anchor_margins(w, bits, gs, im, self.device, sym=sym)
        self._apply_neuqi_anchor(wmin, wmax)

    def _apply_neuqi_anchor(self, wmin, wmax):
        self._applied = (wmin.detach().clone(), wmax.detach().clone())
        self.weight_min = wmin
        self.weight_max = wmax
        self._deferred_neuqi_anchor = None


class TestBatchedNeuqiAnchors(unittest.TestCase):
    def _make(self, n, **kw):
        return [FakeAnchor(i, **kw) for i in range(n)]

    def test_batched_anchor_matches_serial(self):
        for sym in (True, False):
            for with_imatrix in (True, False):
                serial = self._make(4, sym=sym, with_imatrix=with_imatrix)
                for w in serial:
                    w._run_deferred_neuqi_anchor_now()
                batched = self._make(4, sym=sym, with_imatrix=with_imatrix)
                from auto_round.algorithms.quantization.search_dispatch import run_batched_neuqi_anchors

                self.assertTrue(run_batched_neuqi_anchors(batched))
                for s, b in zip(serial, batched):
                    self.assertTrue(torch.equal(s._applied[0], b._applied[0]), f"sym={sym} im={with_imatrix} wmin")
                    self.assertTrue(torch.equal(s._applied[1], b._applied[1]), f"sym={sym} im={with_imatrix} wmax")

    def test_probe_shrinks_anchor_batches(self):
        from auto_round.algorithms.quantization import search_dispatch

        batched = self._make(8)
        # tiny usable headroom: at most two modules per stacked batch
        from auto_round.data_type.neuqi import _MAX_TMP_ELEMS

        # half the headroom must cover the per-call sweep budget plus two
        # modules' fp32 copy chain (incl. imatrix)
        per_module = 8 * 128 * 4 * 16 + 8 * 128 * 4
        with mock.patch.object(
            search_dispatch,
            "probe_usable_bytes",
            return_value=(_MAX_TMP_ELEMS * 4 * 8 + per_module * 2) * 2 + 1,
        ), mock.patch.dict(os.environ, {"AR_PERF_COUNTERS": "1"}), mock.patch.object(
            search_dispatch.logger, "info"
        ) as dbg:
            self.assertTrue(search_dispatch.run_batched_neuqi_anchors(batched))
        args = dbg.call_args.args
        text = args[0] % tuple(args[1:])
        # each chunk's own transient (two modules' chain + the per-call
        # sweep budget) fits half the probed headroom, serially re-earned
        self.assertIn("4 batch", text)

    def test_execute_time_headroom_splits_chunks(self):
        # plan-time probe blind (cpu-staged weights) -> one big chunk; the
        # executing device then reports too little headroom -> the chunk
        # halves recursively down to per-module calls
        import auto_round.data_type.neuqi as neuqi_mod
        from auto_round.algorithms.quantization import search_dispatch

        calls = {"n": 0}
        real = neuqi_mod.neuqi_anchor_margins

        def _counting(weight, bits, gs, im, device, sym=False):
            calls["n"] += 1
            return real(weight, bits, gs, im, device, sym=sym)

        from auto_round.data_type.neuqi import _MAX_TMP_ELEMS

        per = 8 * 128 * 4 * 16 + 8 * 128 * 4 + _MAX_TMP_ELEMS * 4 * 8
        batched = self._make(8)
        with mock.patch.object(
            search_dispatch, "probe_usable_bytes", side_effect=[None] + [per] * 64
        ), mock.patch.object(neuqi_mod, "neuqi_anchor_margins", _counting):
            self.assertTrue(search_dispatch.run_batched_neuqi_anchors(batched))
        self.assertEqual(calls["n"], 8)  # split down to single-module stacks
        for w in batched:
            self.assertIsNotNone(w._applied)

    def test_search_batch_gb_bounds_anchor_stacks(self):
        # the env budget counts weight + expanded-imatrix elements, exactly
        # like the wrap lane
        import os as _os

        from auto_round.algorithms.quantization import search_dispatch

        batched = self._make(8)  # with imatrix: 2 x 1024 elements per module
        budget_elements = 2048  # one module's weight+imatrix, no room for a pair
        gb = budget_elements * 4 / 2**30
        saved = _os.environ.get("AR_SEARCH_BATCH_GB")
        _os.environ["AR_SEARCH_BATCH_GB"] = repr(gb)
        try:
            with mock.patch.dict(os.environ, {"AR_PERF_COUNTERS": "1"}), mock.patch.object(
                search_dispatch.logger, "info"
            ) as dbg:
                self.assertTrue(search_dispatch.run_batched_neuqi_anchors(batched))
        finally:
            if saved is None:
                _os.environ.pop("AR_SEARCH_BATCH_GB", None)
            else:
                _os.environ["AR_SEARCH_BATCH_GB"] = saved
        args = dbg.call_args.args
        text = args[0] % tuple(args[1:])
        self.assertIn("8 batch", text)  # one module per stack under the tight budget

    def test_disabled_batching_returns_false(self):
        from auto_round.algorithms.quantization import search_dispatch

        batched = self._make(3)
        with mock.patch.object(search_dispatch, "batched_search_disabled", return_value=True):
            self.assertFalse(search_dispatch.run_batched_neuqi_anchors(batched))
        # the caller's per-module fallback then finishes them
        for w in batched:
            w._run_deferred_neuqi_anchor_now()
        self.assertTrue(all(w._applied is not None for w in batched))


class TestWrapSearchExecuteTimeSplit(unittest.TestCase):
    def test_headroom_shortfall_halves_chunks(self):
        # plan-time probe blind (cpu) -> one chunk; the executor then sees too
        # little headroom -> the chunk halves down to per-module stacks
        import auto_round.algorithms.quantization.search_dispatch as sd

        fakes = [FakeDeferred(seed=i, shape=(8, 128)) for i in range(4)]
        calls = []
        real_stack = torch.stack

        def spy_stack(tensors, *a, **k):
            calls.append(len(tensors))
            return real_stack(tensors, *a, **k)

        with mock.patch.object(sd, "probe_usable_bytes", side_effect=[None] + [1] * 64), mock.patch.object(
            torch, "stack", side_effect=spy_stack
        ):
            self.assertTrue(run_batched_wrap_search(fakes))
        self.assertEqual(calls, [1] * 8)  # four single-module stacks x (weights + imatrix)
        for f in fakes:
            self.assertIsNotNone(f.init_scale)


class TestOffloadedImatrixDevice(unittest.TestCase):
    def test_offloaded_chunk_moves_materialized_imatrix_to_worker(self):
        # cross-device tensor pairs cannot be hosted on a CPU-only test box;
        # pin the source contract instead (the materialized imatrix stack is
        # built from home-resident staged columns and must follow the stacked
        # weights onto the executing device before the search call)
        import inspect

        from auto_round.algorithms.quantization import search_dispatch as sd

        src = inspect.getsource(sd.run_batched_wrap_search)
        self.assertIn("stacked_im.to(worker_key", src)