Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
int4 packed: trace the training dequantize inside torch.compile (#14385)
Daniel Han66fb355abf5d
diff --git a/tests/test_compressed_tensors_int4.py b/tests/test_compressed_tensors_int4.pyindex 7512549345a..e8be8994b68 100644--- a/tests/test_compressed_tensors_int4.py+++ b/tests/test_compressed_tensors_int4.py@@ -851,3 +851,56 @@ def test_train_mode_no_grad_forward_skips_the_fused_kernel_cache():assert qs._fast is not Nonelin.train()assert qs._fast is None+++def _plain_linear(group = 128, symmetric = False):+ from unsloth.models.compressed_tensors_int4 import Int4PackedLinear++ torch.manual_seed(0)+ packed, ref, gs = _packed_layer(256, 1024, 4, group, symmetric, False, torch.bfloat16)+ lin = Int4PackedLinear.__new__(Int4PackedLinear)+ torch.nn.Module.__init__(lin)+ lin.in_features, lin.out_features = 1024, 256+ for name, tensor in packed.items():+ lin.register_parameter(name, torch.nn.Parameter(tensor.clone(), requires_grad = False))+ lin.register_parameter("bias", None)+ lin._int4_bits, lin._int4_group_size = 4, gs+ return lin+++@needs_gpu+@needs_ct[email protected]("layout", ["checkpoint", "repacked"])+def test_training_linear_traces_inside_torch_compile(layout):+ import torch._dynamo.utils as dynamo_utils++ if layout == "repacked":+ if not has_real_cuda() or torch.cuda.get_device_capability() < (8, 0):+ pytest.skip("tensor-core layouts need sm_80+")+ lin = _repacked_linear()[0]+ else:+ lin = _plain_linear()+ assert lin.quant_state.layout is None+ lin.train()++ def step(x):+ return lin(x * 1)++ torch._dynamo.reset()+ dynamo_utils.counters.clear()+ compiled = torch.compile(step, fullgraph = True)+ try:+ for rows in (37, 64, 129, 500):+ g = torch.Generator(device = "cuda").manual_seed(rows)+ x = torch.randn(2, rows, 1024, device = "cuda", dtype = torch.bfloat16, generator = g)+ dy = torch.randn(2, rows, 256, device = "cuda", dtype = torch.bfloat16, generator = g)+ xe = x.clone().requires_grad_(True)+ ye = step(xe)+ ye.backward(dy)+ xc = x.clone().requires_grad_(True)+ yc = compiled(xc)+ yc.backward(dy)+ assert torch.equal(yc, ye) and torch.equal(xc.grad, xe.grad), rows+ assert sum(dynamo_utils.counters["graph_break"].values()) == 0+ finally:+ torch._dynamo.reset()diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.pyindex 19de18012a7..5294b868fd9 100644--- a/unsloth/kernels/int4_packed.py+++ b/unsloth/kernels/int4_packed.py@@ -265,14 +265,24 @@ def _launch_args(packed, qs):cached = qs._launchif cached is not None and cached[0] == packed.data_ptr():return cached[1]+ args = _make_launch_args(packed, qs, store = True)+ qs._launch = (packed.data_ptr(), args)+ qs._launchers = None+ return args+++def _make_launch_args(packed, qs, store):N, K = qs.shapebits = qs.bitspf = 32 // bitsgroup = qs.group_size if qs.group_size > 0 else K- if qs.g_idx is None and group % pf != 0:+ g_idx = qs.g_idx+ if g_idx is None and group % pf != 0:# A word would straddle two groups: address the groups per column instead.- qs.g_idx = torch.arange(K, device = packed.device, dtype = torch.int32) // group- zp, g_idx, scale = qs.zero_point, qs.g_idx, qs.scale+ g_idx = torch.arange(K, device = packed.device, dtype = torch.int32) // group+ if store:+ qs.g_idx = g_idx+ zp, scale = qs.zero_point, qs.scaleZ = zp if zp is not None else packedG = g_idx if g_idx is not None else packedargs = (@@ -281,8 +291,6 @@ def _launch_args(packed, qs):(packed.stride(0), scale.stride(0), Z.stride(0) if zp is not None else 0),dict(BITS = bits, PF = pf, GROUP = group, HAS_Z = zp is not None, HAS_G = g_idx is not None),)- qs._launch = (packed.data_ptr(), args)- qs._launchers = Nonereturn args@@ -338,12 +346,21 @@ def int4_dequantize(dtype = dtype or qs.dtype or torch.bfloat16if qs.layout is not None:return _dequant_repacked(packed, qs, dtype, out)- (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = _launch_args(packed, qs)+ compiling = torch.compiler.is_compiling()+ # Traced: data_ptr caches and cached launchers would break the graph.+ (P, S, Z, G), (N, K, KP), (sp, ss, sz), meta = (+ _make_launch_args(packed, qs, store = False) if compiling else _launch_args(packed, qs)+ )if out is None:out = torch.empty((N, K), dtype = dtype, device = packed.device)BLOCK_N, BLOCK_KP = 16, max(1, 512 // meta["PF"])grid = (triton.cdiv(N, BLOCK_N), triton.cdiv(KP, BLOCK_KP), 1)args = (P, S, Z, G, out, N, K, KP, sp, ss, sz, out.stride(0))+ if compiling:+ _dequant_kernel[grid](+ *args, BLOCK_N = BLOCK_N, BLOCK_KP = BLOCK_KP, num_warps = 4, **meta, enable_fp_fusion = True+ )+ return outkey = (packed.device.index, "dq", out.dtype, out.data_ptr() % 16 == 0, out.stride(0) % 16 == 0)launchers = qs._launchersif launchers is None: