Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Dequantize block-FP8 weights with a ragged last block on 16-bit loads (GLM-5.3) (#12133)

Daniel Hana9ed05ae2134
diff --git a/tests/test_fp8_ragged_block_dequant.py b/tests/test_fp8_ragged_block_dequant.py
new file mode 100644
index 00000000000..a4f828f6808
--- /dev/null
+++ b/tests/test_fp8_ragged_block_dequant.py
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Block-FP8 weights whose last block is partial (GLM-5.3 / Ling-3.0 kv_a_proj_with_mqa: 576 rows, 5 scale rows) dequantize on 16-bit loads."""
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+fp8 = pytest.importorskip("transformers.integrations.finegrained_fp8")
+if not hasattr(fp8, "Fp8Dequantize"):
+ pytest.skip("transformers 4.x has no Fp8Dequantize, nothing to patch", allow_module_level = True)
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _import_fixes():
+ # By file spec, so unsloth/__init__ (zoo, GPU init) does not run.
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_import_fixes_fp8_ragged_under_test", REPO_ROOT / "unsloth" / "import_fixes.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+class _Quantizer:
+ quantization_config = type("Cfg", (), {"weight_block_size": [128, 128]})()
+
+
+def _op_cls(patched):
+ # A subclass, so patching never leaks into the real class other tests use.
+ cls = type("Fp8DequantizeUnderTest", (fp8.Fp8Dequantize,), {})
+ if patched:
+ _import_fixes()._pad_fp8_dequantize_ragged_blocks(cls)
+ return cls
+
+
+def _dequant(
+ q,
+ s,
+ patched = True,
+):
+ op = _op_cls(patched)(_Quantizer())
+ if hasattr(op, "_dequantize_one"):
+ return op._dequantize_one(q, s, output_dtype = torch.bfloat16)
+ return op.convert({"weight$": [q], "weight_scale_inv": [s]}, full_layer_name = "w")["w"]
+
+
+def _reference(q, s):
+ sf = s.float().repeat_interleave(128, -2)[..., : q.shape[-2], :]
+ sf = sf.repeat_interleave(128, -1)[..., : q.shape[-1]]
+ return (q.float() * sf).to(torch.bfloat16)
+
+
+def _case(*shape, seed = 0):
+ g = torch.Generator().manual_seed(seed)
+ q = (torch.randn(*shape, generator = g) * 50).to(torch.float8_e4m3fn)
+ grid = (*shape[:-2], -(-shape[-2] // 128), -(-shape[-1] // 128))
+ return q, torch.rand(*grid, generator = g) + 0.1
+
+
[email protected]("cols", [384, 2560], ids = ["glm5_3", "ling3"])
+def test_ragged_rows_match_block_reference(cols):
+ q, s = _case(576, cols)
+ with pytest.raises(ValueError):
+ _dequant(q, s, patched = False)
+ out = _dequant(q, s)
+ assert out.shape == (576, cols)
+ assert torch.equal(out.to(torch.bfloat16), _reference(q, s))
+
+
+def test_ragged_rows_already_cast_weight():
+ q, s = _case(576, 2560, seed = 2)
+ assert torch.equal(_dequant(q.to(torch.bfloat16), s).to(torch.bfloat16), _reference(q, s))
+
+
+def test_ragged_columns_and_stacked_experts():
+ # 200 columns over 2 scale columns divides exactly, so transformers 5.17 would use a 100-wide block.
+ q, s = _case(2, 256, 200, seed = 3)
+ out = _dequant(q, s)
+ assert out.shape == (2, 256, 200)
+ assert torch.equal(out.to(torch.bfloat16), _reference(q, s))
+
+
+def test_divisible_shapes_unchanged_and_idempotent():
+ q, s = _case(512, 384, seed = 1)
+ assert torch.equal(_dequant(q, s), _dequant(q, s, patched = False))
+ cls = _op_cls(patched = True)
+ attr = "_dequantize_one" if hasattr(cls, "_dequantize_one") else "convert"
+ wrapped = getattr(cls, attr)
+ _import_fixes()._pad_fp8_dequantize_ragged_blocks(cls)
+ assert getattr(cls, attr) is wrapped and wrapped._unsloth_ragged_blocks
+
+
+def test_grid_that_is_not_a_ceil_tiling_still_raises():
+ q, _ = _case(576, 256)
+ with pytest.raises(ValueError):
+ _dequant(q, torch.ones(7, 2)) # no block edge gives ceil(576 / b) == 7
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index a08d93fd317..59284a6c430 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2788,6 +2788,11 @@ def fix_transformers_fp8_modulelist_experts():
_cast_fp8_dequantize_to_model_dtype(getattr(fp8_integration, "Fp8Dequantize", None))
except Exception:
pass
+ try:
+ import transformers.integrations.finegrained_fp8 as fp8_integration
+ _pad_fp8_dequantize_ragged_blocks(getattr(fp8_integration, "Fp8Dequantize", None))
+ except Exception:
+ pass
return method(self, model, *args, **kwargs)
_process_model_before_weight_loading._unsloth_modulelist_experts = True
@@ -2825,6 +2830,92 @@ def _cast_fp8_dequantize_to_model_dtype(op_cls):
op_cls.convert = cast_convert
+def _fp8_pad_ragged_block(quantized, scales, block):
+ """Zero-pad a block-FP8 weight with a ragged last block to its ceil scale grid, else None."""
+ import torch
+
+ try:
+ rows, cols = quantized.shape[-2:]
+ scale_rows, scale_cols = scales.shape[-2:]
+ block_m, block_n = int(block[0]), int(block[1])
+ except Exception:
+ return None
+ # Packed FP4 doubles its columns inside transformers; FP8 may arrive already cast, so exclude only packed.
+ packed = (torch.int8, torch.uint8, getattr(torch, "float4_e2m1fn_x2", torch.int8))
+ if quantized.dtype in packed or (rows % block_m == 0 and cols % block_n == 0):
+ return None
+ if -(-rows // block_m) != scale_rows or -(-cols // block_n) != scale_cols:
+ return None
+ padded = quantized.new_zeros(
+ (*quantized.shape[:-2], scale_rows * block_m, scale_cols * block_n)
+ )
+ padded[..., :rows, :cols] = quantized
+ return padded
+
+
+def _pad_fp8_dequantize_ragged_blocks(op_cls):
+ """16-bit loads of block-FP8 weights with a ragged last block (GLM-5.3 kv_a_proj_with_mqa: 576 rows) raise: pad, dequantize, slice."""
+ if op_cls is None:
+ return
+
+ def _block(self):
+ config = getattr(getattr(self, "hf_quantizer", None), "quantization_config", None)
+ return getattr(config, "weight_block_size", None)
+
+ original = getattr(op_cls, "_dequantize_one", None)
+ if original is not None:
+ # rows // scale_rows: raises on a ragged grid, or silently picks the wrong block when it divides (200 / 2).
+ if getattr(original, "_unsloth_ragged_blocks", False):
+ return
+
+ @functools.wraps(original)
+ def _dequantize_one(self, quantized, scales, *args, **kwargs):
+ block = _block(self)
+ padded = None if block is None else _fp8_pad_ragged_block(quantized, scales, block)
+ if padded is None:
+ return original(self, quantized, scales, *args, **kwargs)
+ rows, cols = quantized.shape[-2:]
+ out = original(self, padded, scales, *args, **kwargs)
+ return out[..., :rows, :cols].contiguous()
+
+ _dequantize_one._unsloth_ragged_blocks = True
+ op_cls._dequantize_one = _dequantize_one
+ return
+
+ convert = getattr(op_cls, "convert", None)
+ if convert is None or getattr(convert, "_unsloth_ragged_blocks", False):
+ return
+
+ @functools.wraps(convert)
+ def ragged_convert(self, input_dict, *args, **kwargs):
+ try:
+ weight = input_dict["weight$"]
+ scale = input_dict["weight_scale_inv"]
+ quantized = weight[0] if isinstance(weight, list) else weight
+ scales = scale[0] if isinstance(scale, list) else scale
+ block = _block(self)
+ padded = None if block is None else _fp8_pad_ragged_block(quantized, scales, block)
+ except Exception:
+ padded = None
+ if padded is None:
+ return convert(self, input_dict, *args, **kwargs)
+ rows, cols = quantized.shape[-2:]
+ input_dict = dict(input_dict)
+ input_dict["weight$"] = [padded] if isinstance(weight, list) else padded
+ out = convert(self, input_dict, *args, **kwargs)
+ for name, value in out.items():
+ tensor = value[0] if isinstance(value, list) and value else value
+ if getattr(tensor, "shape", None) is not None and tuple(tensor.shape[-2:]) == tuple(
+ padded.shape[-2:]
+ ):
+ tensor = tensor[..., :rows, :cols].contiguous()
+ out[name] = [tensor] if isinstance(value, list) else tensor
+ return out
+
+ ragged_convert._unsloth_ragged_blocks = True
+ op_cls.convert = ragged_convert
+
+
def fix_transformers_is_torch_fx_available():
"""Restore ``is_torch_fx_available`` (removed in 5.0) for 4.x-era remote code."""
try: