Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kernels: name every tl.constexpr parameter with a trailing comment (#16585)
Daniel Han5a13ca9c17a6
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.pyindex 0604a9e9a17..ce33b138b29 100644--- a/studio/backend/core/inference/diffusion_convrot_quant.py+++ b/studio/backend/core/inference/diffusion_convrot_quant.py@@ -159,16 +159,16 @@ def _kernels() -> Optional[types.SimpleNamespace]:s_ptr,M,NG,- G: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_K: tl.constexpr,- ROWS: tl.constexpr,- ROWS_P2: tl.constexpr,- QMIN: tl.constexpr,- QMAX: tl.constexpr,- DIV: tl.constexpr,- EPS: tl.constexpr,- FP32_SCALE: tl.constexpr,+ G: tl.constexpr, # rotation group width+ BLOCK_M: tl.constexpr, # group rows per program+ BLOCK_K: tl.constexpr, # K step inside a group+ ROWS: tl.constexpr, # activation rows per program (from K)+ ROWS_P2: tl.constexpr, # ROWS next pow2+ QMIN: tl.constexpr, # int8 clamp min+ QMAX: tl.constexpr, # int8 clamp max+ DIV: tl.constexpr, # amax divisor (127)+ EPS: tl.constexpr, # norm epsilon, per model+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)):# one program = ROWS whole activation rows, so the per-row amax closes inside the tilepid = tl.program_id(0)@@ -215,9 +215,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:live,h,h32,- TM: tl.constexpr,- SWIGLU: tl.constexpr,- TF32X3: tl.constexpr,+ TM: tl.constexpr, # 256-wide groups per tile+ SWIGLU: tl.constexpr, # fuse swiglu before rotation+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy):# X[a, b] = x[16 a + b]; Y = X . H16 on tensor cores (H16 entries +-1/4: exact products), Z = H16 . Y in fp32.offs = tl.arange(0, 256)@@ -241,9 +241,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:q_ptr,gidx,live,- QMIN: tl.constexpr,- QMAX: tl.constexpr,- FP32_SCALE: tl.constexpr,+ QMIN: tl.constexpr, # int8 clamp min+ QMAX: tl.constexpr, # int8 clamp max+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)):offs = tl.arange(0, 256)p = z * inv_t[:, None]@@ -265,10 +265,10 @@ def _kernels() -> Optional[types.SimpleNamespace]:NG,h,h32,- CG: tl.constexpr,- ROWS: tl.constexpr,- SWIGLU: tl.constexpr,- TF32X3: tl.constexpr,+ CG: tl.constexpr, # groups per chunk+ ROWS: tl.constexpr, # activation rows per program (from K)+ SWIGLU: tl.constexpr, # fuse swiglu before rotation+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy):live = row < Mgidx = tl.where(live, row * NG + c * CG + gl, 0)@@ -289,13 +289,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:NG,h,h32,- CG: tl.constexpr,- ROWS: tl.constexpr,- SWIGLU: tl.constexpr,- TF32X3: tl.constexpr,- QMIN: tl.constexpr,- QMAX: tl.constexpr,- FP32_SCALE: tl.constexpr,+ CG: tl.constexpr, # groups per chunk+ ROWS: tl.constexpr, # activation rows per program (from K)+ SWIGLU: tl.constexpr, # fuse swiglu before rotation+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy+ QMIN: tl.constexpr, # int8 clamp min+ QMAX: tl.constexpr, # int8 clamp max+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)):live = row < Mgidx = tl.where(live, row * NG + c * CG + gl, 0)@@ -312,18 +312,18 @@ def _kernels() -> Optional[types.SimpleNamespace]:s_ptr,M,NG,- NCH: tl.constexpr,- CG: tl.constexpr,- ROWS: tl.constexpr,- ONE_PASS: tl.constexpr,- UNROLL: tl.constexpr,- SWIGLU: tl.constexpr,- TF32X3: tl.constexpr,- QMIN: tl.constexpr,- QMAX: tl.constexpr,- DIV: tl.constexpr,- EPS: tl.constexpr,- FP32_SCALE: tl.constexpr,+ NCH: tl.constexpr, # chunks per row+ CG: tl.constexpr, # groups per chunk+ ROWS: tl.constexpr, # activation rows per program (from K)+ ONE_PASS: tl.constexpr, # whole row fits one chunk+ UNROLL: tl.constexpr, # unroll the chunk loop+ SWIGLU: tl.constexpr, # fuse swiglu before rotation+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy+ QMIN: tl.constexpr, # int8 clamp min+ QMAX: tl.constexpr, # int8 clamp max+ DIV: tl.constexpr, # amax divisor (127)+ EPS: tl.constexpr, # norm epsilon, per model+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)):# ONE_PASS: rotate, amax, quantize from registers. Else an amax pass, then recompute + quant (row re-read from L2).TM: tl.constexpr = ROWS * CGdiff --git a/studio/backend/core/inference/diffusion_flux2_rope.py b/studio/backend/core/inference/diffusion_flux2_rope.pyindex 1c91a2bbab5..e6e09ea81a0 100644--- a/studio/backend/core/inference/diffusion_flux2_rope.py+++ b/studio/backend/core/inference/diffusion_flux2_rope.py@@ -71,8 +71,8 @@ def _kernel() -> Optional[Callable]:sob,sos,soh,- HALF: tl.constexpr,- BLOCK: tl.constexpr,+ HALF: tl.constexpr, # hd // 2+ BLOCK: tl.constexpr, # block size over heads x pairs):row = tl.program_id(0)b = row // Sdiff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex 1a0057b8d73..c4b5499d04f 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -107,12 +107,12 @@ def _kernels() -> Optional[types.SimpleNamespace]:b_ptr,offs,mask,- HAS_BIAS: tl.constexpr,- WS_FP32: tl.constexpr,- EVICT: tl.constexpr,- FINAL_ROUND: tl.constexpr,- FP32_SCALE: tl.constexpr,- PIN_RN: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ WS_FP32: tl.constexpr, # weight scale is fp32+ EVICT: tl.constexpr, # load eviction policy+ FINAL_ROUND: tl.constexpr, # round to bf16 at the end+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)):# Epilogue as torchao: (int32 * x_scale) to bf16, * w_scale, + bias, then bf16; FINAL_ROUND=False defers the last round (it commutes with row max).c = tl.load(c_ptr + offs, mask = mask, other = 0, eviction_policy = EVICT)@@ -161,14 +161,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:p_sb,p_ss,p_sh,- HAS_BIAS: tl.constexpr,- WS_FP32: tl.constexpr,- FP32_SCALE: tl.constexpr,- PIN_RN: tl.constexpr,- HAS_PREFIX: tl.constexpr,- HEAD_DIM: tl.constexpr,- CHUNK: tl.constexpr,- CHUNK_A: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ WS_FP32: tl.constexpr, # weight scale is fp32+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)+ HAS_PREFIX: tl.constexpr, # fused prefix (attn output) columns+ HEAD_DIM: tl.constexpr, # hd of the prefix+ CHUNK: tl.constexpr, # columns per loop step+ CHUNK_A: tl.constexpr, # prefix columns per loop step):row = tl.program_id(0).to(tl.int64)xs = tl.load(xs_ptr + row).to(tl.float32)@@ -301,11 +301,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:v0,offs,m,- HAS_BIAS: tl.constexpr,- WS_FP32: tl.constexpr,- EVICT: tl.constexpr,- FP32_SCALE: tl.constexpr,- PIN_RN: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ WS_FP32: tl.constexpr, # weight scale is fp32+ EVICT: tl.constexpr, # load eviction policy+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)):g = _pre(crow + g0,@@ -351,11 +351,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:V0,stride_c,stride_q,- HAS_BIAS: tl.constexpr,- WS_FP32: tl.constexpr,- FP32_SCALE: tl.constexpr,- PIN_RN: tl.constexpr,- CHUNK: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ WS_FP32: tl.constexpr, # weight scale is fp32+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)+ CHUNK: tl.constexpr, # columns per loop step):# Pass 1 parks the bf16 product in a scratch row: recomputing two int32 rows is ALU-bound.row = tl.program_id(0).to(tl.int64)@@ -413,9 +413,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:K,stride_x,stride_q,- BLOCK: tl.constexpr,- ONE_PASS: tl.constexpr,- FP32_SCALE: tl.constexpr,+ BLOCK: tl.constexpr, # block size along the row+ ONE_PASS: tl.constexpr, # row fits one block+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)):# Matches torchao from_hp PerRow: torchao <=0.17 rounds s, 1/s and x*(1/s) to bf16; >=0.18 keeps them fp32; q = clamp(rint, -128, 127).row = tl.program_id(0).to(tl.int64)@@ -455,7 +455,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:tl.store(qrow + k + offs, p.to(tl.int8), mask = m)@triton.jit- def gelu_bf16(x_ptr, o_ptr, n, BLOCK: tl.constexpr):+ def gelu_bf16(x_ptr, o_ptr, n, BLOCK: tl.constexpr): # block size along the rowoffs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)m = offs < ny = tl.load(x_ptr + offs, mask = m, other = 0.0).to(tl.float32)@@ -469,11 +469,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:b_ptr,o_ptr,N,- HAS_BIAS: tl.constexpr,- WS_FP32: tl.constexpr,- FP32_SCALE: tl.constexpr,- PIN_RN: tl.constexpr,- CHUNK: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ WS_FP32: tl.constexpr, # weight scale is fp32+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)+ CHUNK: tl.constexpr, # columns per loop step):row = tl.program_id(0).to(tl.int64)xs = tl.load(xs_ptr + row).to(tl.float32)diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.pyindex c14488cd940..5de7eaf3159 100644--- a/studio/backend/core/inference/diffusion_int8_gemm.py+++ b/studio/backend/core/inference/diffusion_int8_gemm.py@@ -197,14 +197,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:stride_am,stride_wn,stride_cm,- HAS_BIAS: tl.constexpr,- XS_FP32: tl.constexpr,- WS_FP32: tl.constexpr,- EVEN_K: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- GROUP_M: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ XS_FP32: tl.constexpr, # activation scale is fp32+ WS_FP32: tl.constexpr, # weight scale is fp32+ EVEN_K: tl.constexpr, # K divisible by BLOCK_K+ BLOCK_M: tl.constexpr, # block size along M (rows)+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ GROUP_M: tl.constexpr, # M tiles per swizzle group):pid = tl.program_id(0)num_pid_m = tl.cdiv(M, BLOCK_M)@@ -268,13 +268,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:N,K,stride_cm,- HAS_BIAS: tl.constexpr,- XS_FP32: tl.constexpr,- WS_FP32: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- GROUP_M: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ XS_FP32: tl.constexpr, # activation scale is fp32+ WS_FP32: tl.constexpr, # weight scale is fp32+ BLOCK_M: tl.constexpr, # block size along M (rows)+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ GROUP_M: tl.constexpr, # M tiles per swizzle group):pid = tl.program_id(0)num_pid_m = tl.cdiv(M, BLOCK_M)diff --git a/studio/backend/core/inference/diffusion_int8_gemv.py b/studio/backend/core/inference/diffusion_int8_gemv.pyindex 985d84ce663..ae69bedbf03 100644--- a/studio/backend/core/inference/diffusion_int8_gemv.py+++ b/studio/backend/core/inference/diffusion_int8_gemv.py@@ -64,12 +64,12 @@ def _kernel() -> Any:stride_sn,stride_on,stride_ym,- HAS_BIAS: tl.constexpr,- HAS_OUT: tl.constexpr,- GROUPED: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,+ HAS_BIAS: tl.constexpr, # bias present+ HAS_OUT: tl.constexpr, # bf16 outlier columns present+ GROUPED: tl.constexpr, # per-group weight scales+ BLOCK_M: tl.constexpr, # block size along M (rows)+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)):pid = tl.program_id(0)rn = pid * BLOCK_N + tl.arange(0, BLOCK_N)diff --git a/studio/backend/core/inference/diffusion_moss_fused.py b/studio/backend/core/inference/diffusion_moss_fused.pyindex 0ef8633c155..f1ca2795bec 100644--- a/studio/backend/core/inference/diffusion_moss_fused.py+++ b/studio/backend/core/inference/diffusion_moss_fused.py@@ -174,8 +174,8 @@ def _qk_rope_kernel() -> Any:c2,s2,mask,- EPS: tl.constexpr,- D: tl.constexpr,+ EPS: tl.constexpr, # norm epsilon, per model+ D: tl.constexpr, # hd):# rotate(x) = [-x2, x1]; each product and sum rounded to bf16 like eagerx1 = tl.load(x_ptr + base + h * D + j, mask = mask, other = 0.0).to(tl.float32)@@ -211,9 +211,9 @@ def _qk_rope_kernel() -> Any:svb,svs,scs,- EPS: tl.constexpr,- D: tl.constexpr,- HB: tl.constexpr,+ EPS: tl.constexpr, # norm epsilon, per model+ D: tl.constexpr, # hd+ HB: tl.constexpr, # heads per program):row = tl.program_id(0)b = row // Sdiff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.pyindex cd229fe1ca3..1bb254142a2 100644--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py@@ -50,10 +50,10 @@ if triton is not None:q_ptr,sf_ptr,M,- K: tl.constexpr,- NCB: tl.constexpr,- ROWS: tl.constexpr,- G: tl.constexpr,+ K: tl.constexpr, # in features+ NCB: tl.constexpr, # scale tile columns (K/16/4)+ ROWS: tl.constexpr, # rows per program+ G: tl.constexpr, # 16-element blocks per program):# One program: ROWS rows x G blocks of 16 elements, held as [ROWS, G, 8] even / odd halves so each packed# byte is (even | odd << 4) with no reshape or split (layout conversions some tile shapes miscompiled).diff --git a/studio/backend/core/inference/diffusion_nvfp4_bias.py b/studio/backend/core/inference/diffusion_nvfp4_bias.pyindex 52900b0fdef..a8b0aa35235 100644--- a/studio/backend/core/inference/diffusion_nvfp4_bias.py+++ b/studio/backend/core/inference/diffusion_nvfp4_bias.py@@ -29,7 +29,9 @@ _FAST_BIAS_MAX_NUMEL = 2**31 - 1if _HAVE_TRITON:@triton.jit- def _bias_add_kernel(ptr, bias_ptr, n_elements, n_cols: "tl.constexpr", BLOCK: "tl.constexpr"):+ def _bias_add_kernel(+ ptr, bias_ptr, n_elements, n_cols: "tl.constexpr", BLOCK: "tl.constexpr"+ ): # n_cols out features, BLOCK block sizepid = tl.program_id(0)offsets = pid * BLOCK + tl.arange(0, BLOCK)mask = offsets < n_elementsdiff --git a/studio/backend/core/inference/diffusion_qwenimage21_eager.py b/studio/backend/core/inference/diffusion_qwenimage21_eager.pyindex 1dd62aea674..43094ef5a56 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_eager.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_eager.py@@ -101,10 +101,10 @@ def _kernel() -> Any:stride_h,f_stride,eps,- HALF: tl.constexpr,- ROWS: tl.constexpr,- REAL_FMA_X: tl.constexpr,- IMAG_FMA_X: tl.constexpr,+ HALF: tl.constexpr, # hd // 2+ ROWS: tl.constexpr, # rows per program+ REAL_FMA_X: tl.constexpr, # fma pairing of real part+ IMAG_FMA_X: tl.constexpr, # fma pairing of imag part):pid = tl.program_id(0).to(tl.int64)rows = pid * ROWS + tl.arange(0, ROWS).to(tl.int64)diff --git a/studio/backend/core/inference/diffusion_rocm_fused.py b/studio/backend/core/inference/diffusion_rocm_fused.pyindex 5f5b6c44e52..efafd4bf875 100644--- a/studio/backend/core/inference/diffusion_rocm_fused.py+++ b/studio/backend/core/inference/diffusion_rocm_fused.py@@ -239,7 +239,20 @@ def _adaln_kernel() -> Optional[Callable]:@triton.jitdef _adaln_fwd(- X, SCALE, SHIFT, OUT, S, D, sxb, sxs, scb, shb, sob, sos, eps, BLOCK: tl.constexpr+ X,+ SCALE,+ SHIFT,+ OUT,+ S,+ D,+ sxb,+ sxs,+ scb,+ shb,+ sob,+ sos,+ eps,+ BLOCK: tl.constexpr, # block size along hidden):row = tl.program_id(0)b = row // Sdiff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.pyindex fb7ba9bdbc0..6d25a121270 100644--- a/studio/backend/core/inference/diffusion_vae_fused.py+++ b/studio/backend/core/inference/diffusion_vae_fused.py@@ -71,7 +71,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:def _gn_partials(x_ptr, bias_ptr, pn_ptr, pmean_ptr, pm2_ptr,T, HW, W, n_chunks, sb, sc, st, sh, sw,- C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr+ C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr # C channels, G groupnorm groups, CPG channels per group, BLOCK_P pixels per program, HAS_BIAS bias present): # fmt: skipchunk = tl.program_id(0)bt = tl.program_id(1)@@ -106,7 +106,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jitdef _gn_combine(- pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr+ pn_ptr,+ pmean_ptr,+ pm2_ptr,+ mean_ptr,+ rstd_ptr,+ n_chunks,+ G,+ eps,+ BLOCK: tl.constexpr, # block size):row = tl.program_id(0)g = row % G@@ -138,7 +146,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:def _gn_apply(x_ptr, out_ptr, mean_ptr, rstd_ptr, w_ptr, b_ptr, ib_ptr,C, T, HW, W, G, sb, sc, st, sh, sw,- CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # CPG channels per group, ACT apply silu, HAS_IN_BIAS input bias present, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skippid_p = tl.program_id(0)bt = tl.program_id(1)@@ -179,8 +187,8 @@ def _kernels() -> Optional[types.SimpleNamespace]:x_ptr, out_ptr, w_ptr, b_ptr, ib_ptr, cache_ptr,C, T, H, W, Ho, Wo, ph, pw, To, front, n_cache, scale, eps,sb, sc, st, sh, sw, cb, cc, ct, ch, cw,- NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr,- REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr, # NORM apply rms norm, ACT apply silu, HAS_BIAS bias present, HAS_IN_BIAS input bias present+ REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # REPLICATE replicate temporal padding, MEAN_SQ mean-square rms form, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skippid_p = tl.program_id(0)bt = tl.program_id(1)@@ -252,7 +260,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jit(do_not_specialize = _SHAPE_ARGS["_bias_residual"])def _bias_residual(o_ptr, ob_ptr, r_ptr, rb_ptr, P, C, T, HW, W, sb, sc, st, sh, sw, inv_scale,- HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # HAS_OB output bias present, HAS_RB residual bias present, SCALE output scale != 1, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skipp = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)@@ -285,7 +293,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:h_ptr, hb_ptr, x_ptr, out_ptr,c, Cin, F, H, W, Fo, rep,hsb, hsc, hst, hsh, hsw, xsb, xsc, xst, xsh, xsw,- TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # TEMPORAL temporal upsample, HAS_HB conv bias present, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skippid_p = tl.program_id(0)bf = tl.program_id(1)@@ -338,7 +346,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:tl.store(out_ptr + o_off, (v + xv).to(out_ptr.dtype.element_ty), mask = m)@triton.jit- def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr):+ def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr): # block sizerow = tl.program_id(0).to(tl.int64)base = s_ptr + row * P_STRIDEm = tl.full([BLOCK], float("-inf"), tl.float32)@@ -365,7 +373,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:def _dup_up_add(o_ptr, x_ptr, C, T_o, H_o, W_o, t_off, repeats,osb, osc, ost, osh, osw, xsb, xsc, xst, xsh, xsw,- FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # FT temporal upsample factor, FS spatial upsample factor, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skippid_p = tl.program_id(0)bt = tl.program_id(1)@@ -399,7 +407,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jit(do_not_specialize = _SHAPE_ARGS["_up_nearest2x"])def _up_nearest2x(x_ptr, out_ptr, C, T_o, H, W, xsb, xsc, xst, xsh, xsw,- INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # INTERLEAVE wan time_conv channel interleave, BLOCK_P pixels per program, BLOCK_C channels per program): # fmt: skip# INTERLEAVE: x is Wan time_conv output (B, 2C, T_o/2, H, W); frame 2t+j takes channels [jC, (j+1)C).pid_p = tl.program_id(0)diff --git a/studio/backend/core/inference/video_minimax_h3_qknorm.py b/studio/backend/core/inference/video_minimax_h3_qknorm.pyindex 03ffac4da86..0b39f02c640 100644--- a/studio/backend/core/inference/video_minimax_h3_qknorm.py+++ b/studio/backend/core/inference/video_minimax_h3_qknorm.py@@ -64,9 +64,9 @@ def _kernel() -> Any:stride_h,cos_stride,eps,- D: tl.constexpr,- HALF: tl.constexpr,- ROWS: tl.constexpr,+ D: tl.constexpr, # hd+ HALF: tl.constexpr, # rotary dim // 2+ ROWS: tl.constexpr, # rows per program):pid = tl.program_id(0).to(tl.int64)rows = pid * ROWS + tl.arange(0, ROWS).to(tl.int64)diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.pyindex c48f3f1a4ad..d44a778ab7b 100644--- a/studio/backend/core/inference/video_minimax_h3_vae.py+++ b/studio/backend/core/inference/video_minimax_h3_vae.py@@ -102,11 +102,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:st,sh,sw,- C: tl.constexpr,- G: tl.constexpr,- CPG: tl.constexpr,- BLOCK_P: tl.constexpr,- HAS_BIAS: tl.constexpr,+ C: tl.constexpr, # channels+ G: tl.constexpr, # groupnorm groups+ CPG: tl.constexpr, # channels per group+ BLOCK_P: tl.constexpr, # pixels per program+ HAS_BIAS: tl.constexpr, # bias present):chunk = tl.program_id(0)bt = tl.program_id(1)@@ -141,7 +141,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jitdef _gn_combine(- pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr+ pn_ptr,+ pmean_ptr,+ pm2_ptr,+ mean_ptr,+ rstd_ptr,+ n_chunks,+ G,+ eps,+ BLOCK: tl.constexpr, # block size):row = tl.program_id(0)g = row % G@@ -194,11 +202,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:st,sh,sw,- CPG: tl.constexpr,- HAS_NORM: tl.constexpr,- HAS_IN_BIAS: tl.constexpr,- BLOCK_W: tl.constexpr,- BLOCK_C: tl.constexpr,+ CPG: tl.constexpr, # channels per group+ HAS_NORM: tl.constexpr, # apply groupnorm+ HAS_IN_BIAS: tl.constexpr, # input bias present+ BLOCK_W: tl.constexpr, # width pixels per program+ BLOCK_C: tl.constexpr, # channels per program):pid_row = tl.program_id(0)pid_w = tl.program_id(1)@@ -255,11 +263,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:st,sh,sw,- C: tl.constexpr,- HAS_OB: tl.constexpr,- HAS_RB: tl.constexpr,- R_SAME: tl.constexpr,- BLOCK_P: tl.constexpr,+ C: tl.constexpr, # channels+ HAS_OB: tl.constexpr, # output bias present+ HAS_RB: tl.constexpr, # residual bias present+ R_SAME: tl.constexpr, # residual same layout (channels_last)+ BLOCK_P: tl.constexpr, # pixels per program):p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)pmask = p < P@@ -294,7 +302,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jitdef _add_rmsnorm(- h_ptr, o_ptr, s_ptr, w_ptr, n_ptr, N, eps, HAS_RES: tl.constexpr, BLOCK_N: tl.constexpr+ h_ptr,+ o_ptr,+ s_ptr,+ w_ptr,+ n_ptr,+ N,+ eps,+ HAS_RES: tl.constexpr,+ BLOCK_N: tl.constexpr, # HAS_RES residual add present, BLOCK_N block size along the row):row = tl.program_id(0).to(tl.int64)cols = tl.arange(0, BLOCK_N)@@ -320,9 +336,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:heads,cs_stride,eps,- HEAD_D: tl.constexpr,- ROT: tl.constexpr,- BLOCK: tl.constexpr,+ HEAD_D: tl.constexpr, # hd+ ROT: tl.constexpr, # rotary dim+ BLOCK: tl.constexpr, # block size):hr = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)rmask = hr < n_head_rows@@ -350,7 +366,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:tl.store(x_ptr + base + d[None, :], y.to(dt), mask = rmask[:, None])@triton.jit- def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr):+ def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr): # block size along the rowrow = tl.program_id(0).to(tl.int64)pid_n = tl.program_id(1)cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)@@ -362,7 +378,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:tl.store(out_ptr + row * N + cols, (hid * act).to(dt), mask = mask)@triton.jit- def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr):+ def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr): # block size along the row# IEEE division + half-to-even rounding: the same codes as the torch path.row = tl.program_id(0).to(tl.int64)cols = tl.arange(0, BLOCK_N)@@ -377,7 +393,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:@triton.jitdef _dequant_epilogue(- acc_ptr, xs_ptr, ws_ptr, bias_ptr, out_ptr, N, HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr+ acc_ptr,+ xs_ptr,+ ws_ptr,+ bias_ptr,+ out_ptr,+ N,+ HAS_BIAS: tl.constexpr,+ BLOCK_N: tl.constexpr, # HAS_BIAS bias present, BLOCK_N block size along the row):row = tl.program_id(0).to(tl.int64)pid_n = tl.program_id(1)diff --git a/studio/backend/core/inference/video_wan_fused.py b/studio/backend/core/inference/video_wan_fused.pyindex 8e73e62f3a1..c15110a59c0 100644--- a/studio/backend/core/inference/video_wan_fused.py+++ b/studio/backend/core/inference/video_wan_fused.py@@ -83,9 +83,9 @@ def _kernels() -> Optional[dict]:stl,stk,sbk,- K_SHIFT: tl.constexpr,- K_SCALE: tl.constexpr,- BLOCK: tl.constexpr,+ K_SHIFT: tl.constexpr, # modulation index of shift+ K_SCALE: tl.constexpr, # modulation index of scale+ BLOCK: tl.constexpr, # block size along hidden):# int64: l * 6 * D passes 2**31 on long clips; int32 wraps to an out-of-bounds temb read.row = tl.program_id(0).to(tl.int64)@@ -110,7 +110,19 @@ def _kernels() -> Optional[dict]:@triton.jitdef _gate_residual(- X, A, T, TBL, OUT, D, L, stb, stl, stk, sbk, K_GATE: tl.constexpr, BLOCK: tl.constexpr+ X,+ A,+ T,+ TBL,+ OUT,+ D,+ L,+ stb,+ stl,+ stk,+ sbk,+ K_GATE: tl.constexpr,+ BLOCK: tl.constexpr, # K_GATE modulation index of gate, BLOCK block size along hidden):# out = fp16(x.float() + a.float() * (tbl[gate] + temb[gate]))row = tl.program_id(0).to(tl.int64) # int64 temb offset, as in _modnorm@@ -130,7 +142,9 @@ def _kernels() -> Optional[dict]:tl.store(OUT + row * D + col, y.to(OUT.dtype.element_ty), mask = mask)@triton.jit- def _affine_norm(X, MEAN, RSTD, W, BIAS, OUT, D, BLOCK: tl.constexpr):+ def _affine_norm(+ X, MEAN, RSTD, W, BIAS, OUT, D, BLOCK: tl.constexpr+ ): # block size along hidden# Matches PyTorch's gamma * (...) + beta, which nvcc contracts to one fma.row = tl.program_id(0)col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)@@ -145,7 +159,16 @@ def _kernels() -> Optional[dict]:@triton.jitdef _rope(- X, COS, SIN, OUT, L, scl, ssl, PAIRS: tl.constexpr, HALF: tl.constexpr, BLOCK: tl.constexpr+ X,+ COS,+ SIN,+ OUT,+ L,+ scl,+ ssl,+ PAIRS: tl.constexpr,+ HALF: tl.constexpr,+ BLOCK: tl.constexpr, # PAIRS rotary pairs per row, HALF hd // 2, BLOCK block size along hidden):# Each product a separately rounded fp32 multiply, then one fp16 rounding, matching stock.row = tl.program_id(0)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 087a758ec21..af1b54b4a77 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -59,12 +59,12 @@ def _cross_entropy_forward(loss_ptr,logsumexp_ptr,labels_ptr,- VOCAB_SIZE: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- DO_SOFTCAPPING: tl.constexpr,- SOFTCAP: tl.constexpr,- DO_LOGIT_SCALING: tl.constexpr,- LOGIT_SCALE: tl.constexpr,+ VOCAB_SIZE: tl.constexpr, # vocab size+ BLOCK_SIZE: tl.constexpr, # vocab block size+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on+ SOFTCAP: tl.constexpr, # softcap value+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on+ LOGIT_SCALE: tl.constexpr, # logit scale value):"""Cross Entropy Loss = 1/n sum [ -yi log(Pi) ]@@ -186,19 +186,19 @@ def _chunked_forward_config(device, vocab_size):def _chunked_cross_entropy_forward(logits_ptr,- logits_row_stride: tl.constexpr,+ logits_row_stride: tl.constexpr, # logits row stride (vocab)loss_ptr,logsumexp_ptr,logsumexp_others_ptr,labels_ptr,- VOCAB_SIZE: tl.constexpr,- N_CHUNKS: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- DO_SOFTCAPPING: tl.constexpr,- SOFTCAP: tl.constexpr,- DO_LOGIT_SCALING: tl.constexpr,- LOGIT_SCALE: tl.constexpr,- EVICT: tl.constexpr,+ VOCAB_SIZE: tl.constexpr, # vocab size+ N_CHUNKS: tl.constexpr, # vocab chunks per row+ BLOCK_SIZE: tl.constexpr, # vocab block size+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on+ SOFTCAP: tl.constexpr, # softcap value+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on+ LOGIT_SCALE: tl.constexpr, # logit scale value+ EVICT: tl.constexpr, # logits load eviction policy):"""256K vocab divided in 4 chunks@@ -298,19 +298,19 @@ _chunked_cross_entropy_forward = triton.heuristics(def _cross_entropy_backward(logits_ptr,- logits_row_stride: tl.constexpr,+ logits_row_stride: tl.constexpr, # logits row stride (vocab)dlogits_ptr,dloss_ptr,- dloss_row_stride: tl.constexpr,+ dloss_row_stride: tl.constexpr, # dloss row stridelogsumexp_ptr,loss_ptr,labels_ptr,- VOCAB_SIZE: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- DO_SOFTCAPPING: tl.constexpr,- SOFTCAP: tl.constexpr,- DO_LOGIT_SCALING: tl.constexpr,- LOGIT_SCALE: tl.constexpr,+ VOCAB_SIZE: tl.constexpr, # vocab size+ BLOCK_SIZE: tl.constexpr, # vocab block size+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on+ SOFTCAP: tl.constexpr, # softcap value+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on+ LOGIT_SCALE: tl.constexpr, # logit scale value):"""CE_i = -y log(P) = y * (log[sum(exp(x))] - x)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 39e591a81f1..00593ffefd4 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -197,8 +197,8 @@ def weight_dequant_kernel(y_ptr,M,N,- BLOCK_SIZE: tl.constexpr,- EXACT_F16: tl.constexpr = False,+ BLOCK_SIZE: tl.constexpr, # fp8 quant block size+ EXACT_F16: tl.constexpr = False, # exact fp16 rounding of output):pid_m = tl.program_id(axis = 0)pid_n = tl.program_id(axis = 1)@@ -381,8 +381,8 @@ def act_quant_kernel(x_ptr,y_ptr,s_ptr,- BLOCK_SIZE: tl.constexpr,- LONG_INDEXING: tl.constexpr = False,+ BLOCK_SIZE: tl.constexpr, # fp8 quant block size+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31):pid = tl.program_id(axis = 0)if LONG_INDEXING:@@ -448,11 +448,11 @@ def _w8a8_block_fp8_matmul(stride_As_k,stride_Bs_k,stride_Bs_n,- BLOCK_SIZE_M: tl.constexpr,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- GROUP_SIZE_M: tl.constexpr,- LONG_INDEXING: tl.constexpr = False,+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ GROUP_SIZE_M: tl.constexpr, # M tiles per swizzle group+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31):"""Triton-accelerated function used to perform linear operations (dotproduct) on input tensors `A` and `B` with block-wise quantization, and@@ -542,12 +542,12 @@ def _w8a8_block_fp8_matmul_tma(stride_As_k,stride_Bs_k,stride_Bs_n,- BLOCK_SIZE_M: tl.constexpr,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- GROUP_SIZE_M: tl.constexpr,- NUM_STAGES: tl.constexpr,- OUT_DTYPE: tl.constexpr,+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ GROUP_SIZE_M: tl.constexpr, # M tiles per swizzle group+ NUM_STAGES: tl.constexpr, # pipeline stages+ OUT_DTYPE: tl.constexpr, # output dtype):# _w8a8_block_fp8_matmul with A, B and C moved by TMA descriptors (out-of-range rows and K tail# are zero filled on load, clipped on store) and a pipelined K loop; same per-tile math and order,@@ -851,11 +851,11 @@ def _w8a8_block_fp8_small_m(stride_Bs_n,stride_Bs_k,KB_PER_SPLIT,- GROUP_N: tl.constexpr,- BLOCK_SIZE_M: tl.constexpr,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- SPLIT_K: tl.constexpr,+ GROUP_N: tl.constexpr, # weight scale block along N+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ SPLIT_K: tl.constexpr, # split-K factor):pid_n = tl.program_id(axis = 0)pid_k = tl.program_id(axis = 1)@@ -1208,9 +1208,9 @@ def _fp8_rowwise_gemv_kernel(N,K,stride_wn,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- LONG_INDEXING: tl.constexpr = False,+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31):pid = tl.program_id(0)# Widen before the multiply: row * stride_wn overflows int32 past 2^31 weight elements.@@ -1254,11 +1254,11 @@ def _fp8_rowwise_gemv_rows_kernel(N,K,stride_wn,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- SPLIT_K: tl.constexpr,- LONG_INDEXING: tl.constexpr = False,+ BLOCK_M: tl.constexpr, # rows per program+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ SPLIT_K: tl.constexpr, # split-K factor+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31):pid = tl.program_id(0)pid_k = tl.program_id(1)@@ -1297,7 +1297,13 @@ def _fp8_rowwise_gemv_rows_kernel(@triton.jit(do_not_specialize = ["M"])def _fp8_rowwise_gemv_rows_reduce(- p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr+ p_ptr,+ s_ptr,+ y_ptr,+ M,+ N,+ SPLIT_K: tl.constexpr,+ BLOCK: tl.constexpr, # SPLIT_K split-K factor, BLOCK block size):offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)mask = offs < M * N@@ -1453,12 +1459,12 @@ def _fp8_block_gemv_kernel(stride_sk,stride_ys,stride_ym,- GROUP_N: tl.constexpr,- GROUP_K: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- LONG_INDEXING: tl.constexpr = False,+ GROUP_N: tl.constexpr, # weight scale block along N+ GROUP_K: tl.constexpr, # weight scale block along K+ BLOCK_M: tl.constexpr, # rows per program+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31):pid_n = tl.program_id(0)pid_k = tl.program_id(1)@@ -1713,12 +1719,12 @@ def _e4m3fn_bytes_dequant_kernel(stride_xn,stride_sm,stride_sn,- BLOCK_M_Q: tl.constexpr,- BLOCK_N_Q: tl.constexpr,- BLOCK_R: tl.constexpr,- BLOCK_C: tl.constexpr,- LONG_INDEXING: tl.constexpr,- EXACT_F16: tl.constexpr = False,+ BLOCK_M_Q: tl.constexpr, # scale block rows+ BLOCK_N_Q: tl.constexpr, # scale block cols+ BLOCK_R: tl.constexpr, # rows per program+ BLOCK_C: tl.constexpr, # cols per program+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ EXACT_F16: tl.constexpr = False, # exact fp16 rounding of output):pid_r = tl.program_id(0)pid_c = tl.program_id(1)@@ -1754,11 +1760,11 @@ def _e4m3fn_scale_dequant_kernel(M,N,stride_s,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- SCALE_ON_COLS: tl.constexpr,- ROUND_SCALE: tl.constexpr,- LONG_INDEXING: tl.constexpr,+ BLOCK_M: tl.constexpr, # rows per program+ BLOCK_N: tl.constexpr, # block size along N (out features)+ SCALE_ON_COLS: tl.constexpr, # scale per column (transposed)+ ROUND_SCALE: tl.constexpr, # round scale to output dtype+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31):# Scale per tensor (stride_s 0), row or column. ROUND_SCALE mirrors x.to(dtype) * s.to(dtype) (fp16 keeps# the fp32 scale); e4m3 times a bf16 scale is exact in fp32, so one rounding gives torch's bits.@@ -2233,9 +2239,9 @@ def _fp8_act_quant_per_row_kernel(ub_ptr,K,stride_xm,- HAS_UB: tl.constexpr,- BLOCK_K: tl.constexpr,- ONE_BLOCK: tl.constexpr,+ HAS_UB: tl.constexpr, # scale upper bound present+ BLOCK_K: tl.constexpr, # block size along K (in features)+ ONE_BLOCK: tl.constexpr, # row fits one BLOCK_K):# Same fp32 math Inductor emits for _quantize_fp8_per_row; [1, BLOCK_K] tile is faster than 1D on sm120.row = tl.program_id(0).to(tl.int64)@@ -2277,7 +2283,7 @@ def _fp8_act_quant_per_row_kernel(@triton.jit-def _fp8_rowwise_dequant_kernel(w_ptr, s_ptr, y_ptr, numel, K, BLOCK: tl.constexpr):+def _fp8_rowwise_dequant_kernel(w_ptr, s_ptr, y_ptr, numel, K, BLOCK: tl.constexpr): # block size# One fp32 multiply, one rounding, as Inductor fuses it (eager x.to(dtype) * s.to(dtype) rounds twice).offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)mask = offs < numeldiff --git a/unsloth/kernels/fused_add_rms_layernorm.py b/unsloth/kernels/fused_add_rms_layernorm.pyindex c0a23a8d441..98423cb1690 100644--- a/unsloth/kernels/fused_add_rms_layernorm.py+++ b/unsloth/kernels/fused_add_rms_layernorm.py@@ -25,10 +25,10 @@ def _add_rms_layernorm_inference(W,X_row_stride,R_row_stride,- n_cols: tl.constexpr,- eps: tl.constexpr,- HAS_RESIDUAL: tl.constexpr,- BLOCK_SIZE: tl.constexpr,+ n_cols: tl.constexpr, # hidden size+ eps: tl.constexpr, # norm epsilon, per model+ HAS_RESIDUAL: tl.constexpr, # add residual before norm+ BLOCK_SIZE: tl.constexpr, # block size, next pow2 of row width):row = tl.program_id(0).to(tl.int64)cols = tl.arange(0, BLOCK_SIZE)diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.pyindex e6884506ac2..992ec705b30 100644--- a/unsloth/kernels/geglu.py+++ b/unsloth/kernels/geglu.py@@ -27,7 +27,12 @@ INT32_SAFETY_BUFFER = NUM_INT32_ELEMENTS - BLOCK_SIZE * SAFE_INT32_BUFFER_MULTIP@triton.jitdef _exact_forward_kernel(- e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+ e,+ g,+ h,+ n_elements,+ BLOCK_SIZE: tl.constexpr,+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31):block_idx = tl.program_id(0)if LONG_INDEXING:@@ -69,7 +74,12 @@ def geglu_exact_forward_kernel(gate, up):@triton.jitdef _exact_backward_kernel(- DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+ DW,+ e,+ g,+ n_elements,+ BLOCK_SIZE: tl.constexpr,+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31):"""f = 1/2 * e * (1 + erf(1/sqrt(2) * e))@@ -136,7 +146,12 @@ def geglu_exact_backward_kernel(DW, e, g):@triton.jitdef _approx_forward_kernel(- e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+ e,+ g,+ h,+ n_elements,+ BLOCK_SIZE: tl.constexpr,+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31):block_idx = tl.program_id(0)if LONG_INDEXING:@@ -180,7 +195,12 @@ def geglu_approx_forward_kernel(gate, up):@triton.jitdef _approx_backward_kernel(- DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+ DW,+ e,+ g,+ n_elements,+ BLOCK_SIZE: tl.constexpr,+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31):"""f = 1/2 * e * (1 + tanh( sqrt(2/pi) * x * (1 + 0.044715 * x^2 ) ))diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.pyindex fefae25ab0c..e99a6029fb5 100644--- a/unsloth/kernels/int4_packed.py+++ b/unsloth/kernels/int4_packed.py@@ -88,11 +88,11 @@ def _unpack_tile3(stride_pn,stride_sn,stride_zn,- BITS: tl.constexpr,- PF: tl.constexpr,- GROUP: tl.constexpr,- HAS_Z: tl.constexpr,- HAS_G: tl.constexpr,+ BITS: tl.constexpr, # bits per weight (4 or 8)+ PF: tl.constexpr, # values packed per int32 word+ GROUP: tl.constexpr, # quant group size along K+ HAS_Z: tl.constexpr, # zero points present+ HAS_G: tl.constexpr, # act-order g_idx present):# fp32 decode of W[rn, words rkp] as [BLOCK_N, BLOCK_KP, PF] (column = word * PF + j).MASK: tl.constexpr = (1 << BITS) - 1@@ -150,13 +150,13 @@ def _unpack_tile(stride_pn,stride_sn,stride_zn,- BITS: tl.constexpr,- PF: tl.constexpr,- GROUP: tl.constexpr,- HAS_Z: tl.constexpr,- HAS_G: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_KP: tl.constexpr,+ BITS: tl.constexpr, # bits per weight (4 or 8)+ PF: tl.constexpr, # values packed per int32 word+ GROUP: tl.constexpr, # quant group size along K+ HAS_Z: tl.constexpr, # zero points present+ HAS_G: tl.constexpr, # act-order g_idx present+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_KP: tl.constexpr, # packed words per program along K):w = _unpack_tile3(P, S, Z, G, rn, rkp, N, K, KP, stride_pn, stride_sn, stride_zn,@@ -179,13 +179,13 @@ def _dequant_kernel(stride_sn,stride_zn,stride_on,- BITS: tl.constexpr,- PF: tl.constexpr,- GROUP: tl.constexpr,- HAS_Z: tl.constexpr,- HAS_G: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_KP: tl.constexpr,+ BITS: tl.constexpr, # bits per weight (4 or 8)+ PF: tl.constexpr, # values packed per int32 word+ GROUP: tl.constexpr, # quant group size along K+ HAS_Z: tl.constexpr, # zero points present+ HAS_G: tl.constexpr, # act-order g_idx present+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_KP: tl.constexpr, # packed words per program along K):pid_n = tl.program_id(0)pid_k = tl.program_id(1)@@ -221,14 +221,14 @@ def _gemv_kernel(stride_sn,stride_zn,stride_ym,- BITS: tl.constexpr,- PF: tl.constexpr,- GROUP: tl.constexpr,- HAS_Z: tl.constexpr,- HAS_G: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_KP: tl.constexpr,- SPLIT_K: tl.constexpr,+ BITS: tl.constexpr, # bits per weight (4 or 8)+ PF: tl.constexpr, # values packed per int32 word+ GROUP: tl.constexpr, # quant group size along K+ HAS_Z: tl.constexpr, # zero points present+ HAS_G: tl.constexpr, # act-order g_idx present+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_KP: tl.constexpr, # packed words per program along K+ SPLIT_K: tl.constexpr, # split-K factor):# Decode: one activation row per program axis 2 (M tiny), CUDA-core reduction.pid_n = tl.program_id(0)@@ -565,13 +565,13 @@ def _dequant_repacked_kernel(stride_sn,stride_zn,stride_on,- MARLIN: tl.constexpr,- TN: tl.constexpr,- TK: tl.constexpr,- R: tl.constexpr,- GROUP: tl.constexpr,- HAS_Z: tl.constexpr,- RAW: tl.constexpr,+ MARLIN: tl.constexpr, # marlin tile layout (else tinygemm)+ TN: tl.constexpr, # tile rows (N)+ TK: tl.constexpr, # tile cols (K)+ R: tl.constexpr, # tiles along K per program+ GROUP: tl.constexpr, # quant group size along K+ HAS_Z: tl.constexpr, # zero points present+ RAW: tl.constexpr, # store raw int codes):# One program: tile row tn, R tiles along K, written as [TN, R * TK] contiguous rows.pid = tl.program_id(0)diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.pyindex f39638b788e..a114dd2cf3b 100644--- a/unsloth/kernels/large_head_attention.py+++ b/unsloth/kernels/large_head_attention.py@@ -78,9 +78,9 @@ def _lh_fwd(H,G,S,- HEAD_DIM: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_M: tl.constexpr, # query rows per program+ BLOCK_N: tl.constexpr, # key rows per step):start_m = tl.program_id(0)off_bh = tl.program_id(1)@@ -147,8 +147,8 @@ def _lh_bwd_pre(sdom,H,S,- HEAD_DIM: tl.constexpr,- BLOCK_M: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_M: tl.constexpr, # query rows per program):start_m = tl.program_id(0)off_bh = tl.program_id(1)@@ -203,9 +203,9 @@ def _lh_bwd_dq(H,G,S,- HEAD_DIM: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_M: tl.constexpr, # query rows per program+ BLOCK_N: tl.constexpr, # key rows per step):start_m = tl.program_id(0)off_bh = tl.program_id(1)@@ -291,9 +291,9 @@ def _lh_bwd_dkdv(HKV,G,S,- HEAD_DIM: tl.constexpr,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_M: tl.constexpr, # query rows per program+ BLOCK_N: tl.constexpr, # key rows per step):# One program owns a kv block of one kv head and loops over its G query heads,# so GQA needs no atomics and no repeated K / V.diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 6942271b3b8..21768362cc2 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -32,11 +32,11 @@ def layernorm_forward(mu,Y_row_stride: tl.constexpr, # n_colsX_row_stride: tl.constexpr, # n_cols- n_cols: tl.constexpr,- eps: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- HAS_BIAS: tl.constexpr = True,+ n_cols: tl.constexpr, # hidden size+ eps: tl.constexpr, # norm epsilon, per model+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # hidden next pow2+ HAS_BIAS: tl.constexpr = True, # bias present):row_idx = tl.program_id(0)if LONG_INDEXING:@@ -70,7 +70,16 @@ def layernorm_forward(@triton.jitdef _layernorm_backward_row(- dY, dX, X, W_row, r, mu, row_idx, col_offsets, mask, n_cols: tl.constexpr+ dY,+ dX,+ X,+ W_row,+ r,+ mu,+ row_idx,+ col_offsets,+ mask,+ n_cols: tl.constexpr, # hidden size):# based on karpathy/llm.c; float32 matches torchtune's Fp32LayerNorm.dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)@@ -95,13 +104,13 @@ def layernorm_backward(b,r,mu,- dY_row_stride: tl.constexpr,- dX_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)+ dX_row_stride: tl.constexpr, # dX row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ n_cols: tl.constexpr, # hidden size+ eps: tl.constexpr, # norm epsilon, per model+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # hidden next pow2):row_idx = tl.program_id(0)if LONG_INDEXING:@@ -136,14 +145,14 @@ def layernorm_backward_dw(dB,n_rows,rows_per_program,- dY_row_stride: tl.constexpr,- dX_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- n_cols: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- TRAIN_W: tl.constexpr,- TRAIN_B: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)+ dX_row_stride: tl.constexpr, # dX row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ n_cols: tl.constexpr, # hidden size+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # hidden next pow2+ TRAIN_W: tl.constexpr, # weight grad partials+ TRAIN_B: tl.constexpr, # bias grad partials):pid = tl.program_id(0)col_offsets = tl.arange(0, BLOCK_SIZE)@@ -180,7 +189,13 @@ def layernorm_backward_dw(@triton.jitdef _layernorm_sum_partials(- partials, out0, out1, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr+ partials,+ out0,+ out1,+ n_programs,+ n_cols,+ BLOCK_ROWS: tl.constexpr,+ BLOCK_COLS: tl.constexpr, # BLOCK_ROWS partial rows per program, BLOCK_COLS columns per program):# Not torch.sum: Inductor reorders it, so eager and compiled bytes would differ.which = tl.program_id(1)diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex 9e2cd97ba6b..5afa5e70753 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -51,21 +51,21 @@ def _grouped_gemm_dX_kernel(dX_ptr,gather_indices_ptr,m_sizes_ptr,- NUM_EXPERTS: tl.constexpr,+ NUM_EXPERTS: tl.constexpr, # number of expertsNUM_TOKENS,- TOPK: tl.constexpr,- N: tl.constexpr,- K: tl.constexpr,+ TOPK: tl.constexpr, # experts per token (top-k)+ N: tl.constexpr, # out features per expert+ K: tl.constexpr, # in features per expertNUM_SMS,- BLOCK_SIZE_M: tl.constexpr,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- PERMUTE_X: tl.constexpr = False,- PERMUTE_Y: tl.constexpr = False,- USE_TMA_LOAD_W: tl.constexpr = False,- USE_TMA_LOAD_dY: tl.constexpr = False,- USE_TMA_STORE: tl.constexpr = False,- FLATTEN: tl.constexpr = True,+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing+ USE_TMA_LOAD_W: tl.constexpr = False, # TMA load for W+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output+ FLATTEN: tl.constexpr = True, # flatten the expert loop) -> None:TOTAL_TOKENS = NUM_TOKENS * TOPKoutput_dtype = dX_ptr.dtype.element_ty@@ -239,21 +239,21 @@ def _grouped_gemm_dW_kernel(m_sizes_ptr,gather_indices_ptr,NUM_TOKENS,- TOPK: tl.constexpr,- NUM_EXPERTS: tl.constexpr,- N: tl.constexpr,- K: tl.constexpr,+ TOPK: tl.constexpr, # experts per token (top-k)+ NUM_EXPERTS: tl.constexpr, # number of experts+ N: tl.constexpr, # out features per expert+ K: tl.constexpr, # in features per expertNUM_SMS,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- BLOCK_SIZE_M: tl.constexpr,- PERMUTE_X: tl.constexpr = False,- PERMUTE_Y: tl.constexpr = False,- USE_TMA_LOAD_dY: tl.constexpr = False,- USE_TMA_LOAD_X: tl.constexpr = False,- USE_TMA_STORE: tl.constexpr = False,- acc_dtype: tl.constexpr = tl.float32,- ACCUMULATE: tl.constexpr = False,+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype+ ACCUMULATE: tl.constexpr = False, # accumulate into existing dW# Seed the fp32 accumulator with the existing tile (load overlaps the main loop) instead of adding# it after the loop. Only for a bf16 / fp16 dW, whose final rounding dwarfs the fp32 order change;# an fp32 dW adds at the end, the same single fp32 add autograd would do.@@ -436,22 +436,22 @@ def _grouped_gemm_dW_per_expert_kernel(m_sizes_ptr,gather_indices_ptr,NUM_TOKENS,- TOPK: tl.constexpr,- NUM_EXPERTS: tl.constexpr,- N: tl.constexpr,- K: tl.constexpr,+ TOPK: tl.constexpr, # experts per token (top-k)+ NUM_EXPERTS: tl.constexpr, # number of experts+ N: tl.constexpr, # out features per expert+ K: tl.constexpr, # in features per expertNUM_SMS,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- BLOCK_SIZE_M: tl.constexpr,- PERMUTE_X: tl.constexpr = False,- PERMUTE_Y: tl.constexpr = False,- USE_TMA_LOAD_dY: tl.constexpr = False,- USE_TMA_LOAD_X: tl.constexpr = False,- USE_TMA_STORE: tl.constexpr = False,- acc_dtype: tl.constexpr = tl.float32,- ACCUMULATE: tl.constexpr = False,- EXPERTS_POW2: tl.constexpr = 1,+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype+ ACCUMULATE: tl.constexpr = False, # accumulate into existing dW+ EXPERTS_POW2: tl.constexpr = 1, # num experts next pow2) -> None:"""dW[e] = dY_e^T X_e with one program per (expert, n tile, k tile).diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.pyindex 6960cb33d90..3c53a2c6abe 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py@@ -23,26 +23,26 @@ def _grouped_gemm_forward_kernel(gather_indices_ptr,topk_weights_ptr,# Constant problem shapes- NUM_EXPERTS: tl.constexpr,+ NUM_EXPERTS: tl.constexpr, # number of expertsNUM_TOKENS,- TOPK: tl.constexpr,- N: tl.constexpr,- K: tl.constexpr,+ TOPK: tl.constexpr, # experts per token (top-k)+ N: tl.constexpr, # out features per expert+ K: tl.constexpr, # in features per expertNUM_SMS,# Tuning params- BLOCK_SIZE_M: tl.constexpr,- BLOCK_SIZE_N: tl.constexpr,- BLOCK_SIZE_K: tl.constexpr,- PERMUTE_X: tl.constexpr = False,- PERMUTE_Y: tl.constexpr = False,- FUSE_MUL_PRE: tl.constexpr = False,- FUSE_MUL_POST: tl.constexpr = False,- USE_FAST_ACCUM: tl.constexpr = False,- USE_TMA_LOAD_W: tl.constexpr = False,- USE_TMA_LOAD_X: tl.constexpr = False,- USE_TMA_STORE: tl.constexpr = False,- acc_dtype: tl.constexpr = tl.float32,- FLATTEN: tl.constexpr = True,+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing+ FUSE_MUL_PRE: tl.constexpr = False, # multiply topk weight before gemm+ FUSE_MUL_POST: tl.constexpr = False, # multiply topk weight after gemm+ USE_FAST_ACCUM: tl.constexpr = False, # fp8 fast accumulation+ USE_TMA_LOAD_W: tl.constexpr = False, # TMA load for W+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype+ FLATTEN: tl.constexpr = True, # flatten the expert loop) -> None:tl.static_assert(K % BLOCK_SIZE_K == 0)diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.pyindex 507766e42ed..871d4c03d60 100644--- a/unsloth/kernels/nvfp4.py+++ b/unsloth/kernels/nvfp4.py@@ -81,10 +81,10 @@ def _nvfp4_dequant_kernel(rows,half_cols,scale_cols,- BLOCK_R: tl.constexpr,- BLOCK_C: tl.constexpr,- SCALE_E4M3_BYTES: tl.constexpr,- INT64_OFFSETS: tl.constexpr,+ BLOCK_R: tl.constexpr, # rows per program+ BLOCK_C: tl.constexpr, # cols per program+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31):r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)if INT64_OFFSETS:@@ -193,12 +193,12 @@ def _nvfp4_weight_tile(stride_pc,stride_sr,stride_sc,- BLOCK_R: tl.constexpr,- BLOCK_K: tl.constexpr,- SCALE_E4M3_BYTES: tl.constexpr,- OUT_DTYPE: tl.constexpr,- INT64_OFFSETS: tl.constexpr,- MASK_PAD: tl.constexpr,+ BLOCK_R: tl.constexpr, # rows per program+ BLOCK_K: tl.constexpr, # block size along K (in features)+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes+ OUT_DTYPE: tl.constexpr, # output dtype+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)):# Rounded like dense: E2M1 * (block scale / global scale) in fp32, then one cast; out-of-range decodes to 0.c = k0 // 2 + tl.arange(0, BLOCK_K // 2)@@ -251,13 +251,13 @@ def _nvfp4_matmul_fwd_kernel(stride_yn,stride_ys,k_per_split,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- GROUP_M: tl.constexpr,- SCALE_E4M3_BYTES: tl.constexpr,- INT64_OFFSETS: tl.constexpr,- MASK_PAD: tl.constexpr,+ BLOCK_M: tl.constexpr, # block size along M (rows)+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ GROUP_M: tl.constexpr, # M tiles per swizzle group+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)):# Split K writes fp32 partials [split, M, N], summed in a fixed order afterwards (deterministic).pid = tl.program_id(0)@@ -339,13 +339,13 @@ def _nvfp4_matmul_dx_kernel(stride_dk,stride_ds,n_per_split,- BLOCK_M: tl.constexpr,- BLOCK_N: tl.constexpr,- BLOCK_K: tl.constexpr,- GROUP_M: tl.constexpr,- SCALE_E4M3_BYTES: tl.constexpr,- INT64_OFFSETS: tl.constexpr,- MASK_PAD: tl.constexpr,+ BLOCK_M: tl.constexpr, # block size along M (rows)+ BLOCK_N: tl.constexpr, # block size along N (out features)+ BLOCK_K: tl.constexpr, # block size along K (in features)+ GROUP_M: tl.constexpr, # M tiles per swizzle group+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)):pid = tl.program_id(0)split_id = tl.program_id(1)@@ -555,7 +555,9 @@ def _fused_config(@triton.jit-def _split_sum_kernel(partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr):+def _split_sum_kernel(+ partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr+): # SPLIT split-K partials, BLOCK block size# Fixed-order sum of the split partials (determinism), one cast.offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)mask = offs < numeldiff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.pyindex 60b0541ab01..44ba7d92dc2 100644--- a/unsloth/kernels/qk_norm_rope.py+++ b/unsloth/kernels/qk_norm_rope.py@@ -56,8 +56,8 @@ def _norm_rope_fwd_block(sin1,cols,eps,- HEAD_DIM: tl.constexpr,- HAS_ROPE_INDICES: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids):HALF = HEAD_DIM // 2hmask = heads < n_heads@@ -108,9 +108,9 @@ def _qk_norm_rope_forward(sin_row_stride,rope_indices,seqlen,- HEAD_DIM: tl.constexpr,- BLOCK_H: tl.constexpr,- HAS_ROPE_INDICES: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_H: tl.constexpr, # heads per program+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids):row = tl.program_id(0).to(tl.int64)hblock = tl.program_id(1)@@ -156,7 +156,7 @@ def _norm_rope_bwd_block(cos1,sin1,cols,- HEAD_DIM: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd):HALF = HEAD_DIM // 2hmask = heads < n_heads@@ -224,9 +224,9 @@ def _qk_norm_rope_backward(sin_row_stride,rope_indices,seqlen,- HEAD_DIM: tl.constexpr,- BLOCK_H: tl.constexpr,- HAS_ROPE_INDICES: tl.constexpr,+ HEAD_DIM: tl.constexpr, # hd+ BLOCK_H: tl.constexpr, # heads per program+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids):row = tl.program_id(0).to(tl.int64)hblock = tl.program_id(1)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 6e77a3d6112..b52af974ce4 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -27,14 +27,14 @@ def _rms_layernorm_forward(X,W,r,- Y_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- W_row_stride: tl.constexpr,- r_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,+ Y_row_stride: tl.constexpr, # Y row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ W_row_stride: tl.constexpr, # W row stride+ r_row_stride: tl.constexpr, # rstd row stride+ n_cols: tl.constexpr, # hidden size (or hd)+ eps: tl.constexpr, # norm epsilon, per model+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2):"""Fast RMS Layernorm kernel@@ -68,22 +68,22 @@ def _rms_layernorm_forward(@triton.jitdef _rms_backward_row(dY,- dY_row_stride: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)dX,- dX_row_stride: tl.constexpr,+ dX_row_stride: tl.constexpr, # dX row stride (hidden)X,- X_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr, # X row stride (hidden)W,r,- r_row_stride: tl.constexpr,+ r_row_stride: tl.constexpr, # rstd row striderow_idx,col_offsets,mask,- n_cols: tl.constexpr,- GEMMA: tl.constexpr,+ n_cols: tl.constexpr, # hidden size (or hd)+ GEMMA: tl.constexpr, # gemma (1 + w) weightdW_acc,- TRAIN_W: tl.constexpr,- FROM_Y: tl.constexpr,+ TRAIN_W: tl.constexpr, # weight grad partials+ FROM_Y: tl.constexpr, # recompute normed from saved Y):# with FROM_Y, X holds Y and normed = Y / W; r still scales dX.dY += row_idx * dY_row_stride@@ -129,18 +129,18 @@ def _rms_layernorm_backward(dW,n_rows,rows_per_program,- dY_row_stride: tl.constexpr,- dX_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- W_row_stride: tl.constexpr,- r_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- GEMMA: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- TRAIN_W: tl.constexpr,- FROM_Y: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)+ dX_row_stride: tl.constexpr, # dX row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ W_row_stride: tl.constexpr, # W row stride+ r_row_stride: tl.constexpr, # rstd row stride+ n_cols: tl.constexpr, # hidden size (or hd)+ eps: tl.constexpr, # norm epsilon, per model+ GEMMA: tl.constexpr, # gemma (1 + w) weight+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2+ TRAIN_W: tl.constexpr, # weight grad partials+ FROM_Y: tl.constexpr, # recompute normed from saved Y):"""Fast RMS Layernorm kernel for the backward pass@@ -219,14 +219,14 @@ def _gemma_rms_layernorm_forward(X,W,r,- Y_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- W_row_stride: tl.constexpr,- r_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,+ Y_row_stride: tl.constexpr, # Y row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ W_row_stride: tl.constexpr, # W row stride+ r_row_stride: tl.constexpr, # rstd row stride+ n_cols: tl.constexpr, # hidden size (or hd)+ eps: tl.constexpr, # norm epsilon, per model+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2):# Copies google-deepmind/gemma layers.py#L31 and keras-nlp gemma/rms_normalization.py#L33 exactly:# essentially all in float32.@@ -255,7 +255,9 @@ def _gemma_rms_layernorm_forward(@triton.jit-def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):+def _fold_lanes(+ x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr+): # ROWS rows per program, WARPS num_warps of the launch, HALF half of a lane pairreturn tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)@@ -265,7 +267,9 @@ _HIP_ROW_ORDER = tl.constexpr(bool(torch.version.hip))@triton.jit-def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+def _row_dot_in_hip_row_order(+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr+): # ROWS rows per program, WARPS num_warps of the launch# Bit-exact replay of the one-row kernels' tl.sum on AMD (WARPS = BLOCK_SIZE // wavefront):# fma(a_l, b_l, a_(l-8) * b_(l-8)), row_shr 4, 2, 1, 16-lane rows by lane bit 4 then 5, warps.LANES: tl.constexpr = a.shape[1] // WARPS@@ -287,7 +291,9 @@ def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):@triton.jit-def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+def _row_dot_in_cuda_row_order(+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr+): # ROWS rows per program, WARPS num_warps of the launch# Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):# fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))@@ -306,7 +312,9 @@ def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):@triton.jit-def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+def _row_dot_in_row_kernel_order(+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr+): # ROWS rows per program, WARPS num_warps of the launchif _HIP_ROW_ORDER:total = _row_dot_in_hip_row_order(a, b, ROWS, WARPS)else:@@ -321,15 +329,15 @@ def _rms_layernorm_forward_rows(W,r,n_rows,- Y_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- GEMMA: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- ROWS: tl.constexpr,- WARPS: tl.constexpr,- LONG_INDEXING: tl.constexpr,+ Y_row_stride: tl.constexpr, # Y row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ n_cols: tl.constexpr, # hidden size (or hd)+ eps: tl.constexpr, # norm epsilon, per model+ GEMMA: tl.constexpr, # gemma (1 + w) weight+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2+ ROWS: tl.constexpr, # rows per program+ WARPS: tl.constexpr, # num_warps of the launch+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31):pid = tl.program_id(0)if LONG_INDEXING:@@ -364,21 +372,21 @@ def _rms_layernorm_forward_rows(@triton.jitdef _rms_backward_rows_block(dY,- dY_row_stride: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)dX,- dX_row_stride: tl.constexpr,+ dX_row_stride: tl.constexpr, # dX row stride (hidden)X,- X_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr, # X row stride (hidden)W,r,rows,n_rows,- n_cols: tl.constexpr,- GEMMA: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- ROWS: tl.constexpr,- WARPS: tl.constexpr,- FROM_Y: tl.constexpr,+ n_cols: tl.constexpr, # hidden size (or hd)+ GEMMA: tl.constexpr, # gemma (1 + w) weight+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2+ ROWS: tl.constexpr, # rows per program+ WARPS: tl.constexpr, # num_warps of the launch+ FROM_Y: tl.constexpr, # recompute normed from saved Y):col_offsets = tl.arange(0, BLOCK_SIZE)row_mask = rows < n_rows@@ -426,18 +434,18 @@ def _rms_layernorm_backward_rows(dW,n_rows,blocks_per_program,- dY_row_stride: tl.constexpr,- dX_row_stride: tl.constexpr,- X_row_stride: tl.constexpr,- n_cols: tl.constexpr,- eps: tl.constexpr,- GEMMA: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- ROWS: tl.constexpr,- WARPS: tl.constexpr,- LONG_INDEXING: tl.constexpr,- TRAIN_W: tl.constexpr,- FROM_Y: tl.constexpr,+ dY_row_stride: tl.constexpr, # dY row stride (hidden)+ dX_row_stride: tl.constexpr, # dX row stride (hidden)+ X_row_stride: tl.constexpr, # X row stride (hidden)+ n_cols: tl.constexpr, # hidden size (or hd)+ eps: tl.constexpr, # norm epsilon, per model+ GEMMA: tl.constexpr, # gemma (1 + w) weight+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2+ ROWS: tl.constexpr, # rows per program+ WARPS: tl.constexpr, # num_warps of the launch+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ TRAIN_W: tl.constexpr, # weight grad partials+ FROM_Y: tl.constexpr, # recompute normed from saved Y):pid = tl.program_id(0)if LONG_INDEXING:@@ -490,7 +498,12 @@ def _rms_layernorm_backward_rows(@triton.jit(do_not_specialize = ("n_programs",))def _rms_sum_dw_partials(- partials, dW, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr+ partials,+ dW,+ n_programs,+ n_cols,+ BLOCK_ROWS: tl.constexpr,+ BLOCK_COLS: tl.constexpr, # BLOCK_ROWS partial rows per step, BLOCK_COLS columns per program):# Fixed tile order, so eager and torch.compile give the same bytes.cols = tl.program_id(0) * BLOCK_COLS + tl.arange(0, BLOCK_COLS)diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex dc5f3962688..57eeb69babe 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -46,12 +46,12 @@ def _rope_heads_tile(n_heads,cos1,sin1,- half_head_dim: tl.constexpr,- HEADS: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- CAST_TO_SIN: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BACKWARD_PASS: tl.constexpr,+ half_head_dim: tl.constexpr, # hd // 2+ HEADS: tl.constexpr, # heads per program+ BLOCK_SIZE: tl.constexpr, # hd // 2 next pow2+ CAST_TO_SIN: tl.constexpr, # compute in sin dtype (fp32 gemma)+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ BACKWARD_PASS: tl.constexpr, # backward (negated sin)):# Rotates HEADS heads of one (batch, seq) row from X into OUT (OUT may be X: each element is# read before it is written by the same program).@@ -123,15 +123,15 @@ def _rope_embedding_QK(n_heads_Q: tl.constexpr, # 32n_heads_K: tl.constexpr, # 8 (0 without K)n_groups_Q: tl.constexpr, # 1- half_head_dim: tl.constexpr,- BACKWARD_PASS: tl.constexpr,- HAS_ROPE_INDICES: tl.constexpr,- LONG_INDEXING: tl.constexpr,- CAST_TO_SIN: tl.constexpr,- HEADS_Q: tl.constexpr,- HEADS_K: tl.constexpr,- BLOCK_SIZE: tl.constexpr,- EVICT_INDICES: tl.constexpr,+ half_head_dim: tl.constexpr, # hd // 2+ BACKWARD_PASS: tl.constexpr, # backward (negated sin)+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31+ CAST_TO_SIN: tl.constexpr, # compute in sin dtype (fp32 gemma)+ HEADS_Q: tl.constexpr, # query heads per program+ HEADS_K: tl.constexpr, # key heads per program+ BLOCK_SIZE: tl.constexpr, # hd // 2 next pow2+ EVICT_INDICES: tl.constexpr, # evict-first hint on position ids):"""RoPE is Q * cos + rotate_half(Q) * sin; the backward is the same with -sin.diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.pyindex 13b2ea33ebc..b82c396c2bf 100644--- a/unsloth/kernels/swiglu.py+++ b/unsloth/kernels/swiglu.py@@ -22,7 +22,9 @@ INT32_SAFETY_BUFFER = NUM_INT32_ELEMENTS - BLOCK_SIZE * SAFE_INT32_BUFFER_MULTIP@triton.jit-def _fg_kernel(e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr):+def _fg_kernel(+ e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+): # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31block_idx = tl.program_id(0)if LONG_INDEXING:offsets = block_idx.to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE).to(tl.int64)@@ -62,7 +64,9 @@ def swiglu_fg_kernel(e, g):@triton.jit-def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr):+def _DWf_DW_dfg_kernel(+ DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr+): # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31"""e = e.float()se = 1.0 / (1.0 + torch.exp(-e))diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.pyindex c32ab1c942b..3cc107abab8 100644--- a/unsloth/optimizers/flash_adamw.py+++ b/unsloth/optimizers/flash_adamw.py@@ -262,7 +262,9 @@ def _variance_update(var, grad, beta2):import triton.language as tl@triton.jit- def kernel(var_ptr, grad_ptr, out_ptr, n, beta2, BLOCK: tl.constexpr):+ def kernel(+ var_ptr, grad_ptr, out_ptr, n, beta2, BLOCK: tl.constexpr+ ): # elements per programoffsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)mask = offsets < nv = tl.load(var_ptr + offsets, mask = mask)