Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Dequantize block-FP8 weights with a ragged last block on 16-bit loads (GLM-5.3) (#12133)
Daniel Hana9ed05ae2134
diff --git a/tests/test_fp8_ragged_block_dequant.py b/tests/test_fp8_ragged_block_dequant.pynew file mode 100644index 00000000000..a4f828f6808--- /dev/null+++ b/tests/test_fp8_ragged_block_dequant.py@@ -0,0 +1,99 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Block-FP8 weights whose last block is partial (GLM-5.3 / Ling-3.0 kv_a_proj_with_mqa: 576 rows, 5 scale rows) dequantize on 16-bit loads."""++import importlib.util+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+fp8 = pytest.importorskip("transformers.integrations.finegrained_fp8")+if not hasattr(fp8, "Fp8Dequantize"):+ pytest.skip("transformers 4.x has no Fp8Dequantize, nothing to patch", allow_module_level = True)++REPO_ROOT = Path(__file__).resolve().parents[1]+++def _import_fixes():+ # By file spec, so unsloth/__init__ (zoo, GPU init) does not run.+ spec = importlib.util.spec_from_file_location(+ "_unsloth_import_fixes_fp8_ragged_under_test", REPO_ROOT / "unsloth" / "import_fixes.py"+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++class _Quantizer:+ quantization_config = type("Cfg", (), {"weight_block_size": [128, 128]})()+++def _op_cls(patched):+ # A subclass, so patching never leaks into the real class other tests use.+ cls = type("Fp8DequantizeUnderTest", (fp8.Fp8Dequantize,), {})+ if patched:+ _import_fixes()._pad_fp8_dequantize_ragged_blocks(cls)+ return cls+++def _dequant(+ q,+ s,+ patched = True,+):+ op = _op_cls(patched)(_Quantizer())+ if hasattr(op, "_dequantize_one"):+ return op._dequantize_one(q, s, output_dtype = torch.bfloat16)+ return op.convert({"weight$": [q], "weight_scale_inv": [s]}, full_layer_name = "w")["w"]+++def _reference(q, s):+ sf = s.float().repeat_interleave(128, -2)[..., : q.shape[-2], :]+ sf = sf.repeat_interleave(128, -1)[..., : q.shape[-1]]+ return (q.float() * sf).to(torch.bfloat16)+++def _case(*shape, seed = 0):+ g = torch.Generator().manual_seed(seed)+ q = (torch.randn(*shape, generator = g) * 50).to(torch.float8_e4m3fn)+ grid = (*shape[:-2], -(-shape[-2] // 128), -(-shape[-1] // 128))+ return q, torch.rand(*grid, generator = g) + 0.1++[email protected]("cols", [384, 2560], ids = ["glm5_3", "ling3"])+def test_ragged_rows_match_block_reference(cols):+ q, s = _case(576, cols)+ with pytest.raises(ValueError):+ _dequant(q, s, patched = False)+ out = _dequant(q, s)+ assert out.shape == (576, cols)+ assert torch.equal(out.to(torch.bfloat16), _reference(q, s))+++def test_ragged_rows_already_cast_weight():+ q, s = _case(576, 2560, seed = 2)+ assert torch.equal(_dequant(q.to(torch.bfloat16), s).to(torch.bfloat16), _reference(q, s))+++def test_ragged_columns_and_stacked_experts():+ # 200 columns over 2 scale columns divides exactly, so transformers 5.17 would use a 100-wide block.+ q, s = _case(2, 256, 200, seed = 3)+ out = _dequant(q, s)+ assert out.shape == (2, 256, 200)+ assert torch.equal(out.to(torch.bfloat16), _reference(q, s))+++def test_divisible_shapes_unchanged_and_idempotent():+ q, s = _case(512, 384, seed = 1)+ assert torch.equal(_dequant(q, s), _dequant(q, s, patched = False))+ cls = _op_cls(patched = True)+ attr = "_dequantize_one" if hasattr(cls, "_dequantize_one") else "convert"+ wrapped = getattr(cls, attr)+ _import_fixes()._pad_fp8_dequantize_ragged_blocks(cls)+ assert getattr(cls, attr) is wrapped and wrapped._unsloth_ragged_blocks+++def test_grid_that_is_not_a_ceil_tiling_still_raises():+ q, _ = _case(576, 256)+ with pytest.raises(ValueError):+ _dequant(q, torch.ones(7, 2)) # no block edge gives ceil(576 / b) == 7diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex a08d93fd317..59284a6c430 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -2788,6 +2788,11 @@ def fix_transformers_fp8_modulelist_experts():_cast_fp8_dequantize_to_model_dtype(getattr(fp8_integration, "Fp8Dequantize", None))except Exception:pass+ try:+ import transformers.integrations.finegrained_fp8 as fp8_integration+ _pad_fp8_dequantize_ragged_blocks(getattr(fp8_integration, "Fp8Dequantize", None))+ except Exception:+ passreturn method(self, model, *args, **kwargs)_process_model_before_weight_loading._unsloth_modulelist_experts = True@@ -2825,6 +2830,92 @@ def _cast_fp8_dequantize_to_model_dtype(op_cls):op_cls.convert = cast_convert+def _fp8_pad_ragged_block(quantized, scales, block):+ """Zero-pad a block-FP8 weight with a ragged last block to its ceil scale grid, else None."""+ import torch++ try:+ rows, cols = quantized.shape[-2:]+ scale_rows, scale_cols = scales.shape[-2:]+ block_m, block_n = int(block[0]), int(block[1])+ except Exception:+ return None+ # Packed FP4 doubles its columns inside transformers; FP8 may arrive already cast, so exclude only packed.+ packed = (torch.int8, torch.uint8, getattr(torch, "float4_e2m1fn_x2", torch.int8))+ if quantized.dtype in packed or (rows % block_m == 0 and cols % block_n == 0):+ return None+ if -(-rows // block_m) != scale_rows or -(-cols // block_n) != scale_cols:+ return None+ padded = quantized.new_zeros(+ (*quantized.shape[:-2], scale_rows * block_m, scale_cols * block_n)+ )+ padded[..., :rows, :cols] = quantized+ return padded+++def _pad_fp8_dequantize_ragged_blocks(op_cls):+ """16-bit loads of block-FP8 weights with a ragged last block (GLM-5.3 kv_a_proj_with_mqa: 576 rows) raise: pad, dequantize, slice."""+ if op_cls is None:+ return++ def _block(self):+ config = getattr(getattr(self, "hf_quantizer", None), "quantization_config", None)+ return getattr(config, "weight_block_size", None)++ original = getattr(op_cls, "_dequantize_one", None)+ if original is not None:+ # rows // scale_rows: raises on a ragged grid, or silently picks the wrong block when it divides (200 / 2).+ if getattr(original, "_unsloth_ragged_blocks", False):+ return++ @functools.wraps(original)+ def _dequantize_one(self, quantized, scales, *args, **kwargs):+ block = _block(self)+ padded = None if block is None else _fp8_pad_ragged_block(quantized, scales, block)+ if padded is None:+ return original(self, quantized, scales, *args, **kwargs)+ rows, cols = quantized.shape[-2:]+ out = original(self, padded, scales, *args, **kwargs)+ return out[..., :rows, :cols].contiguous()++ _dequantize_one._unsloth_ragged_blocks = True+ op_cls._dequantize_one = _dequantize_one+ return++ convert = getattr(op_cls, "convert", None)+ if convert is None or getattr(convert, "_unsloth_ragged_blocks", False):+ return++ @functools.wraps(convert)+ def ragged_convert(self, input_dict, *args, **kwargs):+ try:+ weight = input_dict["weight$"]+ scale = input_dict["weight_scale_inv"]+ quantized = weight[0] if isinstance(weight, list) else weight+ scales = scale[0] if isinstance(scale, list) else scale+ block = _block(self)+ padded = None if block is None else _fp8_pad_ragged_block(quantized, scales, block)+ except Exception:+ padded = None+ if padded is None:+ return convert(self, input_dict, *args, **kwargs)+ rows, cols = quantized.shape[-2:]+ input_dict = dict(input_dict)+ input_dict["weight$"] = [padded] if isinstance(weight, list) else padded+ out = convert(self, input_dict, *args, **kwargs)+ for name, value in out.items():+ tensor = value[0] if isinstance(value, list) and value else value+ if getattr(tensor, "shape", None) is not None and tuple(tensor.shape[-2:]) == tuple(+ padded.shape[-2:]+ ):+ tensor = tensor[..., :rows, :cols].contiguous()+ out[name] = [tensor] if isinstance(value, list) else tensor+ return out++ ragged_convert._unsloth_ragged_blocks = True+ op_cls.convert = ragged_convert++def fix_transformers_is_torch_fx_available():"""Restore ``is_torch_fx_available`` (removed in 5.0) for 4.x-era remote code."""try: