Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

int4 packed: trace the training dequantize inside torch.compile (#14385)

Daniel Han66fb355abf5d
diff --git a/tests/test_compressed_tensors_int4.py b/tests/test_compressed_tensors_int4.py
index 7512549345a..e8be8994b68 100644
--- a/tests/test_compressed_tensors_int4.py
+++ b/tests/test_compressed_tensors_int4.py
@@ -851,3 +851,56 @@ def test_train_mode_no_grad_forward_skips_the_fused_kernel_cache():
assert qs._fast is not None
lin.train()
assert qs._fast is None
+
+
+def _plain_linear(group = 128, symmetric = False):
+ from unsloth.models.compressed_tensors_int4 import Int4PackedLinear
+
+ torch.manual_seed(0)
+ packed, ref, gs = _packed_layer(256, 1024, 4, group, symmetric, False, torch.bfloat16)
+ lin = Int4PackedLinear.__new__(Int4PackedLinear)
+ torch.nn.Module.__init__(lin)
+ lin.in_features, lin.out_features = 1024, 256
+ for name, tensor in packed.items():
+ lin.register_parameter(name, torch.nn.Parameter(tensor.clone(), requires_grad = False))
+ lin.register_parameter("bias", None)
+ lin._int4_bits, lin._int4_group_size = 4, gs
+ return lin
+
+
+@needs_gpu
+@needs_ct
[email protected]("layout", ["checkpoint", "repacked"])
+def test_training_linear_traces_inside_torch_compile(layout):
+ import torch._dynamo.utils as dynamo_utils
+
+ if layout == "repacked":
+ if not has_real_cuda() or torch.cuda.get_device_capability() < (8, 0):
+ pytest.skip("tensor-core layouts need sm_80+")
+ lin = _repacked_linear()[0]
+ else:
+ lin = _plain_linear()
+ assert lin.quant_state.layout is None
+ lin.train()
+
+ def step(x):
+ return lin(x * 1)
+
+ torch._dynamo.reset()
+ dynamo_utils.counters.clear()
+ compiled = torch.compile(step, fullgraph = True)
+ try:
+ for rows in (37, 64, 129, 500):
+ g = torch.Generator(device = "cuda").manual_seed(rows)
+ x = torch.randn(2, rows, 1024, device = "cuda", dtype = torch.bfloat16, generator = g)
+ dy = torch.randn(2, rows, 256, device = "cuda", dtype = torch.bfloat16, generator = g)
+ xe = x.clone().requires_grad_(True)
+ ye = step(xe)
+ ye.backward(dy)
+ xc = x.clone().requires_grad_(True)
+ yc = compiled(xc)
+ yc.backward(dy)
+ assert torch.equal(yc, ye) and torch.equal(xc.grad, xe.grad), rows
+ assert sum(dynamo_utils.counters["graph_break"].values()) == 0
+ finally:
+ torch._dynamo.reset()
diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.py
index 19de18012a7..5294b868fd9 100644
--- a/unsloth/kernels/int4_packed.py
+++ b/unsloth/kernels/int4_packed.py
@@ -265,14 +265,24 @@ def _launch_args(packed, qs):
cached = qs._launch
if cached is not None and cached[0] == packed.data_ptr():
return cached[1]
+ args = _make_launch_args(packed, qs, store = True)
+ qs._launch = (packed.data_ptr(), args)
+ qs._launchers = None
+ return args
+
+
+def _make_launch_args(packed, qs, store):
N, K = qs.shape
bits = qs.bits
pf = 32 // bits
group = qs.group_size if qs.group_size > 0 else K
- if qs.g_idx is None and group % pf != 0:
+ g_idx = qs.g_idx
+ if g_idx is None and group % pf != 0:
# A word would straddle two groups: address the groups per column instead.
- qs.g_idx = torch.arange(K, device = packed.device, dtype = torch.int32) // group
- zp, g_idx, scale = qs.zero_point, qs.g_idx, qs.scale
+ g_idx = torch.arange(K, device = packed.device, dtype = torch.int32) // group
+ if store:
+ qs.g_idx = g_idx
+ zp, scale = qs.zero_point, qs.scale
Z = zp if zp is not None else packed
G = g_idx if g_idx is not None else packed
args = (
@@ -281,8 +291,6 @@ def _launch_args(packed, qs):
(packed.stride(0), scale.stride(0), Z.stride(0) if zp is not None else 0),
dict(BITS = bits, PF = pf, GROUP = group, HAS_Z = zp is not None, HAS_G = g_idx is not None),
)
- qs._launch = (packed.data_ptr(), args)
- qs._launchers = None
return args
@@ -338,12 +346,21 @@ def int4_dequantize(
dtype = dtype or qs.dtype or torch.bfloat16
if qs.layout is not None:
return _dequant_repacked(packed, qs, dtype, out)
- (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = _launch_args(packed, qs)
+ compiling = torch.compiler.is_compiling()
+ # Traced: data_ptr caches and cached launchers would break the graph.
+ (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = (
+ _make_launch_args(packed, qs, store = False) if compiling else _launch_args(packed, qs)
+ )
if out is None:
out = torch.empty((N, K), dtype = dtype, device = packed.device)
BLOCK_N, BLOCK_KP = 16, max(1, 512 // meta["PF"])
grid = (triton.cdiv(N, BLOCK_N), triton.cdiv(KP, BLOCK_KP), 1)
args = (P, S, Z, G, out, N, K, KP, sp, ss, sz, out.stride(0))
+ if compiling:
+ _dequant_kernel[grid](
+ *args, BLOCK_N = BLOCK_N, BLOCK_KP = BLOCK_KP, num_warps = 4, **meta, enable_fp_fusion = True
+ )
+ return out
key = (packed.device.index, "dq", out.dtype, out.data_ptr() % 16 == 0, out.stride(0) % 16 == 0)
launchers = qs._launchers
if launchers is None: