PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[MTP]split gate_up_proj and fix accu gap in rtn quantization (#1758)
Xin Hec859fabc028f
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex e51f1835c..b67372b34 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -47,6 +47,108 @@ from auto_round.logger import loggerfrom auto_round.utils.common import compress_layer_namesfrom auto_round.utils.weight_handler import _dequant_fp8_linear_weight+# ------------------------------------------------------------------ #+# Fused expert projection patterns #+# ------------------------------------------------------------------ #+# Maps a fused projection name to its constituent split names.+# When a 3D tensor ``*.experts.<fused_name>`` (shape [num_experts, ...])+# is encountered, it is split along dimension 0 of the per-expert 2-D+# slice to produce one tensor per split name per expert.+_FUSED_EXPERT_PROJ_PATTERNS: dict[str, list[str]] = {+ "gate_up_proj": ["gate_proj", "up_proj"],+}+++def split_fused_expert_tensors(+ tensors_dict: dict[str, torch.Tensor],+) -> dict[str, torch.Tensor]:+ """Split 3-D fused expert tensors into per-expert 2-D tensors.++ Many MoE checkpoints store expert weights as fused 3-D parameters+ under ``*.experts.<proj_name>`` (for example ``gate_up_proj`` with shape+ ``[num_experts, 2*intermediate, hidden]``, or ``down_proj`` with shape+ ``[num_experts, out, in]``). After unfusing, each expert gets its own+ 2-D weight tensor, e.g.+ ``experts.0.gate_proj.weight [intermediate, hidden]``.++ Splitting rules:++ * **gate_up_proj** ``[N, 2*inter, hidden]`` →+ ``experts.{i}.gate_proj.weight`` + ``experts.{i}.up_proj.weight``+ * **up_gate_proj** ``[N, 2*inter, hidden]`` →+ ``experts.{i}.up_proj.weight`` + ``experts.{i}.gate_proj.weight``+ * **Other** stacked projections (e.g. ``down_proj``) ``[N, out, in]`` →+ ``experts.{i}.<proj>.weight``++ Non-3-D or non-expert tensors pass through unchanged.++ Args:+ tensors_dict: Mapping of tensor names to tensors.++ Returns:+ New dict with fused expert tensors replaced by per-expert 2-D tensors.+ """+ result: dict[str, torch.Tensor] = {}+ split_count = 0++ for tensor_name, tensor in tensors_dict.items():+ if tensor.dim() != 3:+ result[tensor_name] = tensor+ continue++ # Strip optional .weight suffix for pattern matching+ stripped = tensor_name+ stripped = stripped.removesuffix(".weight") # len(".weight") == 7++ # Expect: <prefix>.experts.<proj_name>+ dot_idx = stripped.rfind(".")+ if dot_idx < 0:+ result[tensor_name] = tensor+ continue++ parent = stripped[:dot_idx]+ proj_name = stripped[dot_idx + 1 :]++ # The immediate parent must be "experts"+ if not parent.endswith("experts") and parent.split(".")[-1] != "experts":+ result[tensor_name] = tensor+ continue++ num_experts = tensor.shape[0]++ if proj_name in _FUSED_EXPERT_PROJ_PATTERNS:+ split_names = _FUSED_EXPERT_PROJ_PATTERNS[proj_name]+ logger.info(+ f"Splitting fused expert tensor '{tensor_name}' "+ f"(shape={list(tensor.shape)}, num_experts={num_experts}) "+ f"into {split_names}"+ )+ for i in range(num_experts):+ expert_2d = tensor[i] # [fused_out, in_features]+ chunks = expert_2d.chunk(len(split_names), dim=0)+ for split_name, chunk in zip(split_names, chunks):+ out_key = f"{parent}.{i}.{split_name}.weight"+ result[out_key] = chunk.contiguous()+ else:+ logger.info(+ f"Splitting stacked expert tensor '{tensor_name}' "+ f"(shape={list(tensor.shape)}, num_experts={num_experts})"+ )+ for i in range(num_experts):+ expert_2d = tensor[i] # [out, in]+ out_key = f"{parent}.{i}.{proj_name}.weight"+ result[out_key] = expert_2d.contiguous()++ split_count += 1++ if split_count:+ logger.info(+ f"Split {split_count} fused expert tensor(s) into "+ f"{len(result) - (len(tensors_dict) - split_count)} per-expert tensors."+ )++ return result+def copy_missing_tensors_from_source(source_dir: str,@@ -271,6 +373,12 @@ def copy_missing_tensors_from_source()for tensor_name in _iter_tensors:missing_tensors_dict[tensor_name] = f.get_tensor(tensor_name)++ # ------------------------------------------------------------------ #+ # Split fused expert tensors (3-D) into per-expert 2-D tensors #+ # ------------------------------------------------------------------ #+ missing_tensors_dict = split_fused_expert_tensors(missing_tensors_dict)+# ------------------------------------------------------------------ ## FP8 dequantization: if a weight is FP8 and its scale_inv is present ## ------------------------------------------------------------------ #@@ -399,7 +507,10 @@ def quantize_weight_rtn(out_features, in_features = weight.shapeif device is None:device = weight.device- weight = weight.to(device).float()+ # Single-step transfer + cast avoids an intermediate BF16 copy on CUDA+ # (``weight.to(device).float()`` would briefly allocate both BF16 and+ # float32 buffers on the target device).+ weight = weight.to(device=device, dtype=torch.float32)# --- pad in_features to multiple of group_size ---if in_features % group_size != 0:@@ -420,44 +531,98 @@ def quantize_weight_rtn(# Reshape for group-wise ops: [padded_out, num_groups, group_size]w_grouped = weight.reshape(padded_out, num_groups, group_size)+ q_scale_thresh = 1e-5 # match quant_tensor_sym / quant_tensor_asym threshold+if sym:- half_range = (1 << (bits - 1)) - 1 # e.g. 7 for 4-bit- zero_point = 1 << (bits - 1) # e.g. 8 for 4-bit+ # Full-range symmetric quantization — aligned with quant_tensor_sym+ # in auto_round/data_type/int.py:+ # maxq = 2^(bits-1) (e.g. 8 for 4-bit)+ # scale = signed_max(wmin, wmax) / maxq (can be negative)+ # q = clamp(round(w / scale), -maxq, maxq - 1)+ maxq = 1 << (bits - 1) # e.g. 8 for 4-bit+ zero_point = maxq # unsigned offset for packing- w_absmax = w_grouped.abs().amax(dim=-1, keepdim=True).clamp(min=1e-10)- scales_full = w_absmax / half_range # [padded_out, num_groups, 1]+ w_min = w_grouped.amin(dim=-1, keepdim=True)+ w_max = w_grouped.amax(dim=-1, keepdim=True)+ # clamp so that 0 is always representable+ w_min = torch.clamp(w_min, max=0)+ w_max = torch.clamp(w_max, min=0)+ wmin_abs = -w_min+ wmax_abs = w_max+ # signed max: preserves the sign of the dominant extreme+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)+ del w_min, w_max, wmin_abs, wmax_abs++ scales_full = (max_v / maxq).to(torch.float16)+ # Clamp small scales for numerical stability (match q_scale_thresh)+ scales_full = torch.where(+ scales_full < 0,+ torch.clamp(scales_full, max=-q_scale_thresh),+ torch.clamp(scales_full, min=q_scale_thresh),+ )+ scales_full = scales_full.float() # compute in fp32+ del max_v- q = torch.round(w_grouped / scales_full).clamp(-half_range - 1, half_range)- q = (q + zero_point).to(torch.int32) # shift to unsigned [0, 2^bits-1]+ # In-place ops to minimise peak GPU memory.+ q = w_grouped / scales_full+ del weight, w_grouped # free the (potentially large) float32 weight storage+ q.round_().clamp_(-maxq, maxq - 1)+ q += zero_point # shift to unsigned [0, 2*maxq - 1]+ q = q.to(torch.int32) # new int32 tensor; float32 q is freedzp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)else:+ # Asymmetric quantization — aligned with quant_tensor_asym+ # in auto_round/data_type/int.py:+ # maxq = 2^bits - 1+ # wmin = clamp(min, max=0), wmax = clamp(max, min=0)+ # scale = (wmax - wmin) / maxq+ # zp = round(-wmin / scale)+ # q = clamp(round(w / scale) + zp, 0, maxq)max_int = (1 << bits) - 1- w_min = w_grouped.amin(dim=-1, keepdim=True)- w_max = w_grouped.amax(dim=-1, keepdim=True)-- scales_full = ((w_max - w_min).clamp(min=1e-10)) / max_int- zp_full = torch.round(-w_min / scales_full).clamp(0, max_int).to(torch.int32)-- q = torch.round(w_grouped / scales_full + zp_full.float()).clamp(0, max_int).to(torch.int32)+ w_min = torch.clamp(w_grouped.amin(dim=-1, keepdim=True), max=0)+ w_max = torch.clamp(w_grouped.amax(dim=-1, keepdim=True), min=0)++ scales_full = ((w_max - w_min) / max_int).to(torch.float16)+ scales_full = torch.clamp(scales_full, min=q_scale_thresh)+ scales_full = scales_full.float() # compute in fp32+ zp_full = torch.round(-w_min / scales_full).clamp_(0, max_int).to(torch.int32)+ del w_min, w_max++ q = w_grouped / scales_full+ del weight, w_grouped+ q.round_() # round FIRST (matches regular path: round(w/scale) then + zp)+ q += zp_full.float() # then add zp+ q.clamp_(0, max_int)+ q = q.to(torch.int32)zp = zp_full.squeeze(-1).t().contiguous() # [num_groups, padded_out]+ del zp_full# scales → [num_groups, padded_out] (float16)scales_out = scales_full.squeeze(-1).t().contiguous().to(torch.float16)+ del scales_full# q → [in_features, padded_out]q = q.reshape(padded_out, in_features).t().contiguous()# ---- Pack qweight: [in_features // pack_factor, padded_out] ----- qweight = torch.zeros(in_features // pack_factor, padded_out, dtype=torch.int32, device=device)- for k in range(pack_factor):- qweight |= q[k::pack_factor, :] << (bits * k)+ # Vectorised: reshape → broadcast shift → int64 sum (≡ bitwise-OR for+ # non-overlapping bit lanes) avoids a Python loop per bit-lane.+ _shifts = torch.arange(pack_factor, dtype=torch.int64, device=device) * bits+ q_packed = q.reshape(in_features // pack_factor, pack_factor, padded_out).to(torch.int64)+ del q+ qweight = (q_packed << _shifts[None, :, None]).sum(dim=1).to(torch.int32)+ del q_packed# ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----- qzeros = torch.zeros(num_groups, padded_out // pack_factor, dtype=torch.int32, device=device)- for k in range(pack_factor):- qzeros |= zp[:, k::pack_factor] << (bits * k)+ # The auto_round:auto_gptq format (qlinear_torch_zp) adds +1 to zeros+ # after unpacking, so we must subtract 1 before packing to compensate.+ zp -= 1+ zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)+ del zp+ qzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)+ del zp_packed, _shifts# Remove output padding from qweight / scales (qzeros stays in pack units)if out_pad > 0:diff --git a/test/test_cpu/utils/test_missing_tensors.py b/test/test_cpu/utils/test_missing_tensors.pyindex a167ff908..88dd4a329 100644--- a/test/test_cpu/utils/test_missing_tensors.py+++ b/test/test_cpu/utils/test_missing_tensors.py@@ -15,16 +15,21 @@import jsonimport osimport tempfile-import unittest+import pytestimport torchfrom auto_round.utils.missing_tensors import (_get_woq_config_from_dir,copy_missing_tensors_from_source,quantize_weight_rtn,+ split_fused_expert_tensors,)+# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+def _save_safetensors(tensors: dict, path: str) -> None:from safetensors.torch import save_file@@ -50,621 +55,532 @@ def _write_config(target_dir: str, qcfg: dict | None = None) -> None:json.dump(config, f)-class TestQuantizeWeightRtn(unittest.TestCase):- """Tests for the standalone quantize_weight_rtn helper."""+def _make_auto_round_config(bits=4, group_size=128, sym=True, block_name_to_quantize=None) -> dict:+ qcfg: dict = {+ "quantization_config": {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ }+ }+ if block_name_to_quantize is not None:+ qcfg["quantization_config"]["block_name_to_quantize"] = block_name_to_quantize+ return qcfg- def test_output_shapes_4bit_sym(self):- """Check that output tensors have the expected shapes for 4-bit symmetric quantization."""- out_features, in_features = 32, 64- bits, group_size = 4, 32- weight = torch.randn(out_features, in_features)- pack_factor = 32 // bits # 8- num_groups = in_features // group_size # 2+# ===========================================================================+# split_fused_expert_tensors+# ===========================================================================- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)- self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))- self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))- self.assertEqual(scales.shape, (num_groups, out_features))- self.assertEqual(qweight.dtype, torch.int32)- self.assertEqual(qzeros.dtype, torch.int32)- self.assertEqual(scales.dtype, torch.float16)+class TestSplitFusedExpertTensors:- def test_output_shapes_4bit_asym(self):- """Check output shapes for 4-bit asymmetric quantization."""- out_features, in_features = 32, 64- bits, group_size = 4, 32- weight = torch.randn(out_features, in_features)+ def test_2d_and_non_expert_pass_through(self):+ """2-D tensors and 3-D non-expert tensors are returned unchanged."""+ tensors = {+ "mlp.fc1.weight": torch.randn(64, 128),+ "mlp.fc1.bias": torch.randn(64),+ "model.layers.0.something.weight": torch.randn(4, 8, 16),+ }+ result = split_fused_expert_tensors(tensors)+ assert set(result.keys()) == set(tensors.keys())+ for k in tensors:+ assert torch.equal(result[k], tensors[k])++ def test_empty_dict_returns_empty(self):+ assert split_fused_expert_tensors({}) == {}++ def test_gate_up_proj_split(self):+ """experts.gate_up_proj [N,2I,H] → N*(gate_proj + up_proj)."""+ N, I, H = 4, 64, 32+ fused = torch.randn(N, 2 * I, H)+ result = split_fused_expert_tensors({"model.layers.0.mlp.experts.gate_up_proj": fused})++ assert "model.layers.0.mlp.experts.gate_up_proj" not in result+ assert len(result) == N * 2+ for i in range(N):+ gk = f"model.layers.0.mlp.experts.{i}.gate_proj.weight"+ uk = f"model.layers.0.mlp.experts.{i}.up_proj.weight"+ assert result[gk].shape == (I, H)+ assert result[uk].shape == (I, H)+ assert torch.equal(result[gk], fused[i, :I, :])+ assert torch.equal(result[uk], fused[i, I:, :])++ def test_gate_up_proj_with_weight_suffix(self):+ N, I, H = 2, 16, 8+ fused = torch.randn(N, 2 * I, H)+ result = split_fused_expert_tensors({"model.experts.gate_up_proj.weight": fused})+ assert "model.experts.gate_up_proj.weight" not in result+ for i in range(N):+ assert f"model.experts.{i}.gate_proj.weight" in result+ assert f"model.experts.{i}.up_proj.weight" in result++ def test_stacked_down_proj(self):+ """experts.down_proj [N,O,I] → N * down_proj.weight."""+ N, O, I = 3, 32, 16+ stacked = torch.randn(N, O, I)+ result = split_fused_expert_tensors({"model.layers.0.mlp.experts.down_proj": stacked})+ assert len(result) == N+ for i in range(N):+ key = f"model.layers.0.mlp.experts.{i}.down_proj.weight"+ assert result[key].shape == (O, I)+ assert torch.equal(result[key], stacked[i])++ def test_mixed_fused_and_normal(self):+ N, I, H = 2, 32, 16+ tensors = {+ "mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),+ "mlp.experts.down_proj": torch.randn(N, H, I),+ "attn.q_proj.weight": torch.randn(H, H),+ }+ result = split_fused_expert_tensors(tensors)+ assert "mlp.experts.gate_up_proj" not in result+ assert "mlp.experts.down_proj" not in result+ assert "attn.q_proj.weight" in result+ for i in range(N):+ assert f"mlp.experts.{i}.gate_proj.weight" in result+ assert f"mlp.experts.{i}.down_proj.weight" in result++ def test_output_tensors_are_contiguous(self):+ N, I, H = 2, 32, 16+ result = split_fused_expert_tensors(+ {+ "mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),+ "mlp.experts.down_proj": torch.randn(N, H, I),+ }+ )+ for k, v in result.items():+ assert v.is_contiguous(), f"{k} is not contiguous"+++# ===========================================================================+# quantize_weight_rtn+# ===========================================================================- pack_factor = 32 // bits- num_groups = in_features // group_size- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=False)+class TestQuantizeWeightRtn:- self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))- self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))- self.assertEqual(scales.shape, (num_groups, out_features))+ def test_output_shapes_4bit_sym(self):+ out_f, in_f, bits, gs = 32, 64, 4, 32+ qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)+ pf = 32 // bits+ ng = in_f // gs+ assert qw.shape == (in_f // pf, out_f)+ assert qz.shape == (ng, out_f // pf)+ assert sc.shape == (ng, out_f)+ assert qw.dtype == torch.int32+ assert sc.dtype == torch.float16++ def test_output_shapes_4bit_asym(self):+ out_f, in_f, bits, gs = 32, 64, 4, 32+ qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=False)+ assert qw.shape == (in_f // (32 // bits), out_f)def test_output_shapes_8bit(self):- """Check output shapes for 8-bit symmetric quantization."""- out_features, in_features = 16, 128- bits, group_size = 8, 128- weight = torch.randn(out_features, in_features)-- pack_factor = 32 // bits # 4- num_groups = in_features // group_size # 1-- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)-- self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))- self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))- self.assertEqual(scales.shape, (num_groups, out_features))-- def test_outputs_are_on_cpu(self):- """Verify that all returned tensors are on the CPU regardless of input device."""- weight = torch.randn(16, 64)- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=4, group_size=32)- self.assertEqual(qweight.device.type, "cpu")- self.assertEqual(qzeros.device.type, "cpu")- self.assertEqual(scales.device.type, "cpu")-- def test_raises_on_1d_weight(self):- """quantize_weight_rtn must reject 1-D tensors."""- with self.assertRaises(AssertionError):+ out_f, in_f, bits, gs = 16, 128, 8, 128+ qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)+ pf = 32 // bits+ assert qw.shape == (in_f // pf, out_f)++ def test_outputs_on_cpu(self):+ qw, qz, sc = quantize_weight_rtn(torch.randn(16, 64), bits=4, group_size=32)+ assert qw.device.type == "cpu"+ assert qz.device.type == "cpu"+ assert sc.device.type == "cpu"++ def test_raises_on_1d(self):+ with pytest.raises(AssertionError):quantize_weight_rtn(torch.randn(64), bits=4, group_size=32)def test_in_features_padding(self):- """in_features that is not a multiple of group_size should be padded internally."""- # in_features=70 is not a multiple of group_size=32- out_features, in_features = 16, 70- bits, group_size = 4, 32- weight = torch.randn(out_features, in_features)- # padded in_features = 96 (next multiple of 32)- padded_in = 96- pack_factor = 32 // bits-- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size)- self.assertEqual(qweight.shape, (padded_in // pack_factor, out_features))---class TestGetWoqConfigFromDir(unittest.TestCase):- """Tests for _get_woq_config_from_dir."""-- def test_returns_none_when_no_config(self):- with tempfile.TemporaryDirectory() as d:- self.assertIsNone(_get_woq_config_from_dir(d))-- def test_returns_none_for_wrong_quant_method(self):- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "gptq",- "packing_format": "auto_round:auto_gptq",- "bits": 4,- "group_size": 128,- "sym": True,- },- )- self.assertIsNone(_get_woq_config_from_dir(d))-- def test_returns_none_for_wrong_packing_format(self):- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "auto-round",- "packing_format": "other_format",- "bits": 4,- "group_size": 128,- "sym": True,- },- )- self.assertIsNone(_get_woq_config_from_dir(d))-- def test_returns_none_for_missing_bits(self):- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "auto-round",- "packing_format": "auto_round:auto_gptq",- "group_size": 128,- "sym": True,- },- )- self.assertIsNone(_get_woq_config_from_dir(d))-- def test_returns_none_for_bits_greater_than_8(self):- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "auto-round",- "packing_format": "auto_round:auto_gptq",- "bits": 16,- "group_size": 128,- "sym": True,- },- )- self.assertIsNone(_get_woq_config_from_dir(d))-- def test_valid_config_returns_required_fields(self):- """A fully valid config returns a dict with bits, group_size, sym,- block_name_to_quantize, and extra_config."""- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "auto-round",- "packing_format": "auto_round:auto_gptq",- "bits": 4,- "group_size": 128,- "sym": True,- },- )- cfg = _get_woq_config_from_dir(d)- self.assertIsNotNone(cfg)- self.assertEqual(cfg["bits"], 4)- self.assertEqual(cfg["group_size"], 128)- self.assertTrue(cfg["sym"])- # New fields must also be present- self.assertIn("block_name_to_quantize", cfg)- self.assertIn("extra_config", cfg)-- def test_valid_config_with_extra_config_and_block_name(self):- """extra_config and block_name_to_quantize from config.json are propagated."""- with tempfile.TemporaryDirectory() as d:- _write_config(- d,- {- "quant_method": "auto-round",- "packing_format": "auto_round:auto_gptq",- "bits": 4,- "group_size": 128,- "sym": True,- "block_name_to_quantize": ["model.layers"],- "extra_config": {"mtp.0.fc": {"bits": 8}},- },- )- cfg = _get_woq_config_from_dir(d)- self.assertIsNotNone(cfg)- self.assertEqual(cfg["block_name_to_quantize"], ["model.layers"])- self.assertEqual(cfg["extra_config"], {"mtp.0.fc": {"bits": 8}})---class TestCopyMissingTensorsFromSource(unittest.TestCase):- """End-to-end tests for copy_missing_tensors_from_source.-- Detection is now purely name-based (parent-layer + block-prefix comparison),- so no model loading or mocking is required.-- A source tensor T is "missing" when ALL of:- 1. T is not already present in the saved output,- 2. T's parent layer (rsplit('.', 1)[0]) is absent from the saved parent layers,- 3. T's first numeric block-prefix (e.g. 'mtp.0') is absent from the saved- block-prefixes — OR T has no numeric segment at all.- Special case: 'lm_head.weight' is always excluded.- """-- def _make_auto_round_config(self, bits=4, group_size=128, sym=True, block_name_to_quantize=None) -> dict:- qcfg = {- "quantization_config": {+ qw, _, _ = quantize_weight_rtn(torch.randn(16, 70), bits=4, group_size=32)+ assert qw.shape == (96 // (32 // 4), 16) # padded to 96+++# ===========================================================================+# _get_woq_config_from_dir+# ===========================================================================+++class TestGetWoqConfigFromDir:++ def test_returns_none_when_no_config(self, tmp_path):+ assert _get_woq_config_from_dir(str(tmp_path)) is None++ def test_returns_none_for_wrong_quant_method(self, tmp_path):+ _write_config(+ str(tmp_path),+ {+ "quant_method": "gptq",+ "packing_format": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": 128,+ "sym": True,+ },+ )+ assert _get_woq_config_from_dir(str(tmp_path)) is None++ def test_returns_none_for_wrong_packing_format(self, tmp_path):+ _write_config(+ str(tmp_path),+ {"quant_method": "auto-round", "packing_format": "other", "bits": 4, "group_size": 128, "sym": True},+ )+ assert _get_woq_config_from_dir(str(tmp_path)) is None++ def test_returns_none_for_missing_bits_or_high_bits(self, tmp_path):+ _write_config(+ str(tmp_path),+ {"quant_method": "auto-round", "packing_format": "auto_round:auto_gptq", "group_size": 128, "sym": True},+ )+ assert _get_woq_config_from_dir(str(tmp_path)) is None++ def test_valid_config(self, tmp_path):+ _write_config(+ str(tmp_path),+ {"quant_method": "auto-round","packing_format": "auto_round:auto_gptq",- "bits": bits,- "group_size": group_size,- "sym": sym,- }- }- if block_name_to_quantize is not None:- qcfg["quantization_config"]["block_name_to_quantize"] = block_name_to_quantize- return qcfg-- # ------------------------------------------------------------------ #- # Detection correctness #- # ------------------------------------------------------------------ #-- def test_plain_copy_of_missing_mtp_tensor(self):- """Tensors from blocks entirely absent in the saved output are copied verbatim."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- mtp_norm = torch.randn(64)- _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))-- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(32, 64)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertTrue(os.path.exists(extra_shard), "Extra shard should be created")-- result = _load_safetensors(extra_shard)- self.assertIn("mtp.0.norm.weight", result)- torch.testing.assert_close(result["mtp.0.norm.weight"], mtp_norm)-- def test_tensor_with_known_parent_layer_not_copied(self):- """Source tensor whose parent layer already has tensors in the target is not missing.-- Scenario: source has the original '.weight'; target has the packed '.qweight'- under the same parent layer — the layer was processed, not skipped.- """- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 64)},- os.path.join(source_dir, "model.safetensors"),- )- # Target has the quantised form — same parent layer, different suffix- _save_safetensors(- {"model.layers.0.self_attn.q_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: parent layer is known")-- def test_tensor_with_known_block_prefix_not_copied(self):- """Source tensor whose numeric block-prefix exists in the saved output is not missing.-- Scenario: source has an extra gate weight inside 'model.layers.0'; the target- already contains other tensors from 'model.layers.0' — the block was processed.- """- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {"model.layers.0.mlp.extra_gate.weight": torch.randn(32, 64)},- os.path.join(source_dir, "model.safetensors"),- )- # Target has a different sub-layer inside the same block- _save_safetensors(- {"model.layers.0.mlp.gate_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: block prefix is known")-- def test_lm_head_weight_is_never_copied(self):- """lm_head.weight is always excluded, even when it is absent from the target."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {"lm_head.weight": torch.randn(32, 64)},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(32, 64)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertFalse(os.path.exists(extra_shard), "lm_head.weight must never be copied")-- def test_already_saved_tensors_are_not_duplicated(self):- """Tensors that are already present in the target are not written again."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- mtp_tensor = torch.randn(64)- _save_safetensors(- {"mtp.0.shared_head.norm.weight": mtp_tensor},- os.path.join(source_dir, "model.safetensors"),- )- # Target already has exactly this tensor- _save_safetensors(- {"mtp.0.shared_head.norm.weight": mtp_tensor},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: tensor already present")-- def test_detects_multiple_missing_tensors_from_different_blocks(self):+ "bits": 4,+ "group_size": 128,+ "sym": True,+ },+ )+ cfg = _get_woq_config_from_dir(str(tmp_path))+ assert cfg is not None+ assert cfg["bits"] == 4+ assert "block_name_to_quantize" in cfg+ assert "extra_config" in cfg+++# ===========================================================================+# copy_missing_tensors_from_source+# ===========================================================================+++class TestCopyMissingTensorsFromSource:++ # ----- Detection -----++ def test_plain_copy_of_missing_tensor(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ mtp_norm = torch.randn(64)+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ assert "mtp.0.norm.weight" in result+ torch.testing.assert_close(result["mtp.0.norm.weight"], mtp_norm)++ def test_known_parent_layer_not_copied(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 64)}, os.path.join(src, "model.safetensors")+ )+ _save_safetensors(+ {"model.layers.0.self_attn.q_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_known_block_prefix_not_copied(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {"model.layers.0.mlp.extra_gate.weight": torch.randn(32, 64)}, os.path.join(src, "model.safetensors")+ )+ _save_safetensors(+ {"model.layers.0.mlp.gate_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_lm_head_never_copied(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"lm_head.weight": torch.randn(32, 64)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(32, 64)}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_already_present_not_duplicated(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ t = torch.randn(64)+ _save_safetensors({"mtp.0.norm.weight": t}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"mtp.0.norm.weight": t}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_detects_multiple_missing_tensors_from_different_blocks(self, tmp_path):"""All source tensors from blocks absent in the saved output are copied."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {- "custom_block.0.norm.weight": torch.randn(64),- "mtp.0.norm.weight": torch.randn(64),- "model.embed_tokens.weight": torch.randn(8, 64),- },- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertTrue(os.path.exists(extra_shard))-- result = _load_safetensors(extra_shard)- self.assertIn("custom_block.0.norm.weight", result)- self.assertIn("mtp.0.norm.weight", result)- # Model-known tensor must NOT be in the extra shard- self.assertNotIn("model.embed_tokens.weight", result)-- # ------------------------------------------------------------------ #- # FP8 dequantization #- # ------------------------------------------------------------------ #-- def test_fp8_weight_is_dequantized_to_bf16(self):- """FP8 weights paired with weight_scale_inv are dequantized to BF16 before saving."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- fp8_weight = torch.randn(64, 128).to(torch.float8_e4m3fn)- scale_inv = torch.tensor(0.5)-- _save_safetensors(- {- "mtp.0.ffn.weight": fp8_weight,- "mtp.0.ffn.weight_scale_inv": scale_inv,- },- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertTrue(os.path.exists(extra_shard))-- result = _load_safetensors(extra_shard)- self.assertIn("mtp.0.ffn.weight", result)- # Scale tensor must be dropped after dequantization- self.assertNotIn("mtp.0.ffn.weight_scale_inv", result)- self.assertEqual(result["mtp.0.ffn.weight"].dtype, torch.bfloat16)-- # ------------------------------------------------------------------ #- # WOQ quantization #- # ------------------------------------------------------------------ #-- def test_woq_quantization_replaces_weight_with_packed_tensors(self):- """2-D MTP weight tensors are quantized and packed when an auto-round WOQ config is present."""- out_features, in_features = 32, 128- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- mtp_weight = torch.randn(out_features, in_features)- _save_safetensors(- {"mtp.0.ffn.weight": mtp_weight},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- with open(os.path.join(target_dir, "config.json"), "w") as f:- json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertTrue(os.path.exists(extra_shard))-- result = _load_safetensors(extra_shard)- # Original weight must be replaced by the three packed tensors- self.assertNotIn("mtp.0.ffn.weight", result)- self.assertIn("mtp.0.ffn.qweight", result)- self.assertIn("mtp.0.ffn.qzeros", result)- self.assertIn("mtp.0.ffn.scales", result)-- def test_block_name_to_ignore_not_quantized_in_woq_mode(self):- """Weights matching BLOCK_NAME_TO_IGNORE patterns are copied as-is, not quantized."""- out_features, in_features = 32, 128- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- gate_weight = torch.randn(out_features, in_features)- # "mlp.gate." is in BLOCK_NAME_TO_IGNORE- _save_safetensors(- {"mtp.0.mlp.gate.weight": gate_weight},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- with open(os.path.join(target_dir, "config.json"), "w") as f:- json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertTrue(os.path.exists(extra_shard))-- result = _load_safetensors(extra_shard)- # Weight must be present as-is — not packed- self.assertIn("mtp.0.mlp.gate.weight", result)- self.assertNotIn("mtp.0.mlp.gate.qweight", result)-- def test_config_updated_without_block_name_to_quantize_after_woq(self):- """After WOQ, config.json is updated so the new block appears in block_name_to_quantize."""- out_features, in_features = 32, 128- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {"mtp.0.ffn.weight": torch.randn(out_features, in_features)},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- raw_qcfg = self._make_auto_round_config(bits=4, group_size=128, sym=True)- with open(os.path.join(target_dir, "config.json"), "w") as f:- json.dump(raw_qcfg, f)-- copy_missing_tensors_from_source(source_dir, target_dir)-- with open(os.path.join(target_dir, "config.json")) as f:- updated_cfg = json.load(f)-- qcfg = updated_cfg.get("quantization_config", {})- block_names = qcfg.get("block_name_to_quantize", [])- self.assertTrue(- block_names == [],- f"Expected no block_name_to_quantize, got: {block_names}",- )-- def test_config_updated_with_block_name_to_quantize_after_woq(self):- """After WOQ, config.json is updated so the new block appears in block_name_to_quantize."""- out_features, in_features = 32, 128- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {"mtp.0.ffn.weight": torch.randn(out_features, in_features)},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- raw_qcfg = self._make_auto_round_config(- bits=4, group_size=128, sym=True, block_name_to_quantize=["model.layers"]- )- with open(os.path.join(target_dir, "config.json"), "w") as f:- json.dump(raw_qcfg, f)-- copy_missing_tensors_from_source(source_dir, target_dir)-- with open(os.path.join(target_dir, "config.json")) as f:- updated_cfg = json.load(f)-- qcfg = updated_cfg.get("quantization_config", {})- block_names = qcfg.get("block_name_to_quantize", [])- self.assertTrue(- any("mtp" in b for b in block_names),- f"Expected an 'mtp' block in block_name_to_quantize, got: {block_names}",- )-- def test_config_updated_with_extra_config_for_unquantized_2d_weight(self):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {+ "custom_block.0.norm.weight": torch.randn(64),+ "mtp.0.norm.weight": torch.randn(64),+ "model.embed_tokens.weight": torch.randn(8, 64),+ },+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ extra_shard = os.path.join(tgt, "model_extra_tensors.safetensors")+ assert os.path.exists(extra_shard)+ result = _load_safetensors(extra_shard)+ assert "custom_block.0.norm.weight" in result+ assert "mtp.0.norm.weight" in result+ assert "model.embed_tokens.weight" not in result++ # ----- Fused expert splitting -----++ def test_fused_experts_split_and_copied(self, tmp_path):+ N, I, H = 2, 32, 16+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {+ "model.mtp.0.mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),+ "model.mtp.0.mlp.experts.down_proj": torch.randn(N, H, I),+ },+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors({"model.embed_tokens.weight": torch.randn(10, H)}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ assert "model.mtp.0.mlp.experts.gate_up_proj" not in result+ for i in range(N):+ assert f"model.mtp.0.mlp.experts.{i}.gate_proj.weight" in result+ assert f"model.mtp.0.mlp.experts.{i}.up_proj.weight" in result+ assert f"model.mtp.0.mlp.experts.{i}.down_proj.weight" in result++ def test_fused_experts_woq_quantized(self, tmp_path):+ N, I, H = 2, 32, 16+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {+ "model.mtp.0.mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),+ "model.mtp.0.mlp.experts.down_proj": torch.randn(N, H, I),+ },+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors(+ {"model.layers.0.mlp.fc1.qweight": torch.zeros(H // 8, H, dtype=torch.int32)},+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(+ tgt,+ {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": H,+ "sym": True,+ },+ )+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ for i in range(N):+ assert f"model.mtp.0.mlp.experts.{i}.gate_proj.qweight" in result+ assert f"model.mtp.0.mlp.experts.{i}.down_proj.qweight" in result+ assert "model.mtp.0.mlp.experts.0.gate_proj.weight" not in result++ # ----- FP8 dequantization -----++ def test_fp8_dequantized_to_bf16(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {+ "mtp.0.ffn.weight": torch.randn(64, 128).to(torch.float8_e4m3fn),+ "mtp.0.ffn.weight_scale_inv": torch.tensor(0.5),+ },+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ assert "mtp.0.ffn.weight" in result+ assert "mtp.0.ffn.weight_scale_inv" not in result+ assert result["mtp.0.ffn.weight"].dtype == torch.bfloat16++ # ----- WOQ quantization -----++ def test_woq_quantizes_missing_weights(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.ffn.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(), f)+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ assert "mtp.0.ffn.weight" not in result+ assert "mtp.0.ffn.qweight" in result++ def test_block_name_to_ignore_not_quantized(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.mlp.gate.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(), f)+ copy_missing_tensors_from_source(src, tgt)+ result = _load_safetensors(os.path.join(tgt, "model_extra_tensors.safetensors"))+ assert "mtp.0.mlp.gate.weight" in result+ assert "mtp.0.mlp.gate.qweight" not in result++ # ----- Config / index file updates -----++ def test_config_updated_after_woq(self, tmp_path):+ """block_name_to_quantize is extended with the new mtp block when one already exists."""+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.ffn.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(block_name_to_quantize=["model.layers"]), f)+ copy_missing_tensors_from_source(src, tgt)+ with open(os.path.join(tgt, "config.json")) as f:+ cfg = json.load(f)+ block_names = cfg.get("quantization_config", {}).get("block_name_to_quantize", [])+ assert any("mtp" in b for b in block_names)++ def test_config_updated_without_block_name_to_quantize_after_woq(self, tmp_path):+ """When no block_name_to_quantize is set initially, it stays empty after WOQ."""+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.ffn.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(), f)+ copy_missing_tensors_from_source(src, tgt)+ with open(os.path.join(tgt, "config.json")) as f:+ cfg = json.load(f)+ block_names = cfg.get("quantization_config", {}).get("block_name_to_quantize", [])+ assert block_names == [], f"Expected no block_name_to_quantize, got: {block_names}"++ def test_config_updated_with_block_name_to_quantize_after_woq(self, tmp_path):+ """When block_name_to_quantize is set, the new mtp block is appended."""+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.ffn.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(block_name_to_quantize=["model.layers"]), f)+ copy_missing_tensors_from_source(src, tgt)+ with open(os.path.join(tgt, "config.json")) as f:+ cfg = json.load(f)+ block_names = cfg.get("quantization_config", {}).get("block_name_to_quantize", [])+ assert any("mtp" in b for b in block_names), f"Expected an 'mtp' block, got: {block_names}"++ def test_config_updated_with_extra_config_for_unquantized_2d_weight(self, tmp_path):"""When a 2-D weight is skipped by BLOCK_NAME_TO_IGNORE, extra_config is updatedto mark it as full-precision so the inference stack knows not to dequantize it."""- out_features, in_features = 32, 128- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- # "mlp.gate." is in BLOCK_NAME_TO_IGNORE → not quantized → should go to extra_config- _save_safetensors(- {"mtp.0.mlp.gate.weight": torch.randn(out_features, in_features)},- os.path.join(source_dir, "model.safetensors"),- )- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- with open(os.path.join(target_dir, "config.json"), "w") as f:- json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)-- copy_missing_tensors_from_source(source_dir, target_dir)-- with open(os.path.join(target_dir, "config.json")) as f:- updated_cfg = json.load(f)-- extra_config = updated_cfg.get("quantization_config", {}).get("extra_config", {})- self.assertIn("mtp.0.mlp.gate", extra_config)- layer_cfg = extra_config["mtp.0.mlp.gate"]- self.assertEqual(layer_cfg.get("bits"), 16)- self.assertEqual(layer_cfg.get("data_type"), "fp")-- # ------------------------------------------------------------------ #- # Index file handling #- # ------------------------------------------------------------------ #-- def test_index_json_is_updated_for_sharded_target(self):- """When target uses a sharded index, copied tensor names are added to weight_map."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- mtp_norm = torch.randn(64)- _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))-- saved_shard = "model-00001-of-00001.safetensors"- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, saved_shard),- )- index = {"metadata": {"total_size": 0}, "weight_map": {"model.embed_tokens.weight": saved_shard}}- with open(os.path.join(target_dir, "model.safetensors.index.json"), "w") as f:- json.dump(index, f)- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- with open(os.path.join(target_dir, "model.safetensors.index.json")) as f:- updated_index = json.load(f)-- self.assertIn("mtp.0.norm.weight", updated_index["weight_map"])- self.assertEqual(updated_index["weight_map"]["mtp.0.norm.weight"], "model_extra_tensors.safetensors")-- def test_index_json_is_created_for_single_file_target(self):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.mlp.gate.weight": torch.randn(32, 128)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ with open(os.path.join(tgt, "config.json"), "w") as f:+ json.dump(_make_auto_round_config(), f)+ copy_missing_tensors_from_source(src, tgt)+ with open(os.path.join(tgt, "config.json")) as f:+ cfg = json.load(f)+ extra_config = cfg.get("quantization_config", {}).get("extra_config", {})+ assert "mtp.0.mlp.gate" in extra_config+ layer_cfg = extra_config["mtp.0.mlp.gate"]+ assert layer_cfg.get("bits") == 16+ assert layer_cfg.get("data_type") == "fp"++ def test_index_json_updated_for_sharded_target(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.norm.weight": torch.randn(64)}, os.path.join(src, "model.safetensors"))+ saved_shard = "model-00001-of-00001.safetensors"+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, saved_shard))+ with open(os.path.join(tgt, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {"total_size": 0}, "weight_map": {"model.embed_tokens.weight": saved_shard}}, f)+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ with open(os.path.join(tgt, "model.safetensors.index.json")) as f:+ idx = json.load(f)+ assert idx["weight_map"]["mtp.0.norm.weight"] == "model_extra_tensors.safetensors"++ def test_index_json_is_created_for_single_file_target(self, tmp_path):"""When the target uses a single safetensors file and tensors are copied, a newmodel.safetensors.index.json is created mapping all tensors to their shards."""- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- mtp_norm = torch.randn(64)- _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))-- _save_safetensors(- {"model.embed_tokens.weight": torch.randn(8, 64)},- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- index_path = os.path.join(target_dir, "model.safetensors.index.json")- self.assertTrue(os.path.exists(index_path), "An index.json should be created for single-file targets")-- with open(index_path) as f:- index = json.load(f)-- weight_map = index["weight_map"]- self.assertIn("mtp.0.norm.weight", weight_map)- self.assertIn("model.embed_tokens.weight", weight_map)- self.assertEqual(weight_map["mtp.0.norm.weight"], "model_extra_tensors.safetensors")- self.assertEqual(weight_map["model.embed_tokens.weight"], "model.safetensors")-- def test_tensor_with_known_block_prefix_not_copied_gemma(self):- """Source tensor whose numeric block-prefix exists in the saved output is not missing.-- Scenario: source has an extra gate weight inside 'model.layers.0'; the target- already contains other tensors from 'model.layers.0' — the block was processed.- """- with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:- _save_safetensors(- {- "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "language_model.model.norm.weight": torch.randn(64),- "language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),- "language_model.norm.weight": torch.randn(64),- },- os.path.join(source_dir, "model.safetensors"),- )- # Target has a different sub-layer inside the same block- _save_safetensors(- {- "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64), # llama4- "model.language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64), # gemma- "model.language_model.norm.weight": torch.randn(64),- },- os.path.join(target_dir, "model.safetensors"),- )- _write_config(target_dir)-- copy_missing_tensors_from_source(source_dir, target_dir)-- extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")- self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: block prefix is known")---if __name__ == "__main__":- unittest.main()+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors({"mtp.0.norm.weight": torch.randn(64)}, os.path.join(src, "model.safetensors"))+ _save_safetensors({"model.embed_tokens.weight": torch.randn(8, 64)}, os.path.join(tgt, "model.safetensors"))+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ index_path = os.path.join(tgt, "model.safetensors.index.json")+ assert os.path.exists(index_path), "An index.json should be created for single-file targets"+ with open(index_path) as f:+ index = json.load(f)+ weight_map = index["weight_map"]+ assert "mtp.0.norm.weight" in weight_map+ assert "model.embed_tokens.weight" in weight_map+ assert weight_map["mtp.0.norm.weight"] == "model_extra_tensors.safetensors"+ assert weight_map["model.embed_tokens.weight"] == "model.safetensors"++ def test_known_block_prefix_not_copied_gemma(self, tmp_path):+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")+ os.makedirs(src)+ os.makedirs(tgt)+ _save_safetensors(+ {+ "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),+ "language_model.model.norm.weight": torch.randn(64),+ "language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),+ "language_model.norm.weight": torch.randn(64),+ },+ os.path.join(src, "model.safetensors"),+ )+ _save_safetensors(+ {+ "language_model.model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),+ "model.language_model.layers.0.mlp.gate_proj.weight": torch.randn(32, 64),+ "model.language_model.norm.weight": torch.randn(64),+ },+ os.path.join(tgt, "model.safetensors"),+ )+ _write_config(tgt)+ copy_missing_tensors_from_source(src, tgt)+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))