Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kernels: name every tl.constexpr parameter with a trailing comment (#16585)

Daniel Han5a13ca9c17a6
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
index 0604a9e9a17..ce33b138b29 100644
--- a/studio/backend/core/inference/diffusion_convrot_quant.py
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -159,16 +159,16 @@ def _kernels() -> Optional[types.SimpleNamespace]:
s_ptr,
M,
NG,
- G: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_K: tl.constexpr,
- ROWS: tl.constexpr,
- ROWS_P2: tl.constexpr,
- QMIN: tl.constexpr,
- QMAX: tl.constexpr,
- DIV: tl.constexpr,
- EPS: tl.constexpr,
- FP32_SCALE: tl.constexpr,
+ G: tl.constexpr, # rotation group width
+ BLOCK_M: tl.constexpr, # group rows per program
+ BLOCK_K: tl.constexpr, # K step inside a group
+ ROWS: tl.constexpr, # activation rows per program (from K)
+ ROWS_P2: tl.constexpr, # ROWS next pow2
+ QMIN: tl.constexpr, # int8 clamp min
+ QMAX: tl.constexpr, # int8 clamp max
+ DIV: tl.constexpr, # amax divisor (127)
+ EPS: tl.constexpr, # norm epsilon, per model
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
):
# one program = ROWS whole activation rows, so the per-row amax closes inside the tile
pid = tl.program_id(0)
@@ -215,9 +215,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:
live,
h,
h32,
- TM: tl.constexpr,
- SWIGLU: tl.constexpr,
- TF32X3: tl.constexpr,
+ TM: tl.constexpr, # 256-wide groups per tile
+ SWIGLU: tl.constexpr, # fuse swiglu before rotation
+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy
):
# X[a, b] = x[16 a + b]; Y = X . H16 on tensor cores (H16 entries +-1/4: exact products), Z = H16 . Y in fp32.
offs = tl.arange(0, 256)
@@ -241,9 +241,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:
q_ptr,
gidx,
live,
- QMIN: tl.constexpr,
- QMAX: tl.constexpr,
- FP32_SCALE: tl.constexpr,
+ QMIN: tl.constexpr, # int8 clamp min
+ QMAX: tl.constexpr, # int8 clamp max
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
):
offs = tl.arange(0, 256)
p = z * inv_t[:, None]
@@ -265,10 +265,10 @@ def _kernels() -> Optional[types.SimpleNamespace]:
NG,
h,
h32,
- CG: tl.constexpr,
- ROWS: tl.constexpr,
- SWIGLU: tl.constexpr,
- TF32X3: tl.constexpr,
+ CG: tl.constexpr, # groups per chunk
+ ROWS: tl.constexpr, # activation rows per program (from K)
+ SWIGLU: tl.constexpr, # fuse swiglu before rotation
+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy
):
live = row < M
gidx = tl.where(live, row * NG + c * CG + gl, 0)
@@ -289,13 +289,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:
NG,
h,
h32,
- CG: tl.constexpr,
- ROWS: tl.constexpr,
- SWIGLU: tl.constexpr,
- TF32X3: tl.constexpr,
- QMIN: tl.constexpr,
- QMAX: tl.constexpr,
- FP32_SCALE: tl.constexpr,
+ CG: tl.constexpr, # groups per chunk
+ ROWS: tl.constexpr, # activation rows per program (from K)
+ SWIGLU: tl.constexpr, # fuse swiglu before rotation
+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy
+ QMIN: tl.constexpr, # int8 clamp min
+ QMAX: tl.constexpr, # int8 clamp max
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
):
live = row < M
gidx = tl.where(live, row * NG + c * CG + gl, 0)
@@ -312,18 +312,18 @@ def _kernels() -> Optional[types.SimpleNamespace]:
s_ptr,
M,
NG,
- NCH: tl.constexpr,
- CG: tl.constexpr,
- ROWS: tl.constexpr,
- ONE_PASS: tl.constexpr,
- UNROLL: tl.constexpr,
- SWIGLU: tl.constexpr,
- TF32X3: tl.constexpr,
- QMIN: tl.constexpr,
- QMAX: tl.constexpr,
- DIV: tl.constexpr,
- EPS: tl.constexpr,
- FP32_SCALE: tl.constexpr,
+ NCH: tl.constexpr, # chunks per row
+ CG: tl.constexpr, # groups per chunk
+ ROWS: tl.constexpr, # activation rows per program (from K)
+ ONE_PASS: tl.constexpr, # whole row fits one chunk
+ UNROLL: tl.constexpr, # unroll the chunk loop
+ SWIGLU: tl.constexpr, # fuse swiglu before rotation
+ TF32X3: tl.constexpr, # tf32x3 dot for fp32 accuracy
+ QMIN: tl.constexpr, # int8 clamp min
+ QMAX: tl.constexpr, # int8 clamp max
+ DIV: tl.constexpr, # amax divisor (127)
+ EPS: tl.constexpr, # norm epsilon, per model
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
):
# ONE_PASS: rotate, amax, quantize from registers. Else an amax pass, then recompute + quant (row re-read from L2).
TM: tl.constexpr = ROWS * CG
diff --git a/studio/backend/core/inference/diffusion_flux2_rope.py b/studio/backend/core/inference/diffusion_flux2_rope.py
index 1c91a2bbab5..e6e09ea81a0 100644
--- a/studio/backend/core/inference/diffusion_flux2_rope.py
+++ b/studio/backend/core/inference/diffusion_flux2_rope.py
@@ -71,8 +71,8 @@ def _kernel() -> Optional[Callable]:
sob,
sos,
soh,
- HALF: tl.constexpr,
- BLOCK: tl.constexpr,
+ HALF: tl.constexpr, # hd // 2
+ BLOCK: tl.constexpr, # block size over heads x pairs
):
row = tl.program_id(0)
b = row // S
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index 1a0057b8d73..c4b5499d04f 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -107,12 +107,12 @@ def _kernels() -> Optional[types.SimpleNamespace]:
b_ptr,
offs,
mask,
- HAS_BIAS: tl.constexpr,
- WS_FP32: tl.constexpr,
- EVICT: tl.constexpr,
- FINAL_ROUND: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- PIN_RN: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ EVICT: tl.constexpr, # load eviction policy
+ FINAL_ROUND: tl.constexpr, # round to bf16 at the end
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)
):
# Epilogue as torchao: (int32 * x_scale) to bf16, * w_scale, + bias, then bf16; FINAL_ROUND=False defers the last round (it commutes with row max).
c = tl.load(c_ptr + offs, mask = mask, other = 0, eviction_policy = EVICT)
@@ -161,14 +161,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:
p_sb,
p_ss,
p_sh,
- HAS_BIAS: tl.constexpr,
- WS_FP32: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- PIN_RN: tl.constexpr,
- HAS_PREFIX: tl.constexpr,
- HEAD_DIM: tl.constexpr,
- CHUNK: tl.constexpr,
- CHUNK_A: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)
+ HAS_PREFIX: tl.constexpr, # fused prefix (attn output) columns
+ HEAD_DIM: tl.constexpr, # hd of the prefix
+ CHUNK: tl.constexpr, # columns per loop step
+ CHUNK_A: tl.constexpr, # prefix columns per loop step
):
row = tl.program_id(0).to(tl.int64)
xs = tl.load(xs_ptr + row).to(tl.float32)
@@ -301,11 +301,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
v0,
offs,
m,
- HAS_BIAS: tl.constexpr,
- WS_FP32: tl.constexpr,
- EVICT: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- PIN_RN: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ EVICT: tl.constexpr, # load eviction policy
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)
):
g = _pre(
crow + g0,
@@ -351,11 +351,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
V0,
stride_c,
stride_q,
- HAS_BIAS: tl.constexpr,
- WS_FP32: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- PIN_RN: tl.constexpr,
- CHUNK: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)
+ CHUNK: tl.constexpr, # columns per loop step
):
# Pass 1 parks the bf16 product in a scratch row: recomputing two int32 rows is ALU-bound.
row = tl.program_id(0).to(tl.int64)
@@ -413,9 +413,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:
K,
stride_x,
stride_q,
- BLOCK: tl.constexpr,
- ONE_PASS: tl.constexpr,
- FP32_SCALE: tl.constexpr,
+ BLOCK: tl.constexpr, # block size along the row
+ ONE_PASS: tl.constexpr, # row fits one block
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
):
# Matches torchao from_hp PerRow: torchao <=0.17 rounds s, 1/s and x*(1/s) to bf16; >=0.18 keeps them fp32; q = clamp(rint, -128, 127).
row = tl.program_id(0).to(tl.int64)
@@ -455,7 +455,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
tl.store(qrow + k + offs, p.to(tl.int8), mask = m)
@triton.jit
- def gelu_bf16(x_ptr, o_ptr, n, BLOCK: tl.constexpr):
+ def gelu_bf16(x_ptr, o_ptr, n, BLOCK: tl.constexpr): # block size along the row
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
m = offs < n
y = tl.load(x_ptr + offs, mask = m, other = 0.0).to(tl.float32)
@@ -469,11 +469,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
b_ptr,
o_ptr,
N,
- HAS_BIAS: tl.constexpr,
- WS_FP32: tl.constexpr,
- FP32_SCALE: tl.constexpr,
- PIN_RN: tl.constexpr,
- CHUNK: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ FP32_SCALE: tl.constexpr, # scale kept in fp32 (torchao >= 0.18)
+ PIN_RN: tl.constexpr, # pin round-to-nearest mul/add (sm100+)
+ CHUNK: tl.constexpr, # columns per loop step
):
row = tl.program_id(0).to(tl.int64)
xs = tl.load(xs_ptr + row).to(tl.float32)
diff --git a/studio/backend/core/inference/diffusion_int8_gemm.py b/studio/backend/core/inference/diffusion_int8_gemm.py
index c14488cd940..5de7eaf3159 100644
--- a/studio/backend/core/inference/diffusion_int8_gemm.py
+++ b/studio/backend/core/inference/diffusion_int8_gemm.py
@@ -197,14 +197,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:
stride_am,
stride_wn,
stride_cm,
- HAS_BIAS: tl.constexpr,
- XS_FP32: tl.constexpr,
- WS_FP32: tl.constexpr,
- EVEN_K: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- GROUP_M: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ XS_FP32: tl.constexpr, # activation scale is fp32
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ EVEN_K: tl.constexpr, # K divisible by BLOCK_K
+ BLOCK_M: tl.constexpr, # block size along M (rows)
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ GROUP_M: tl.constexpr, # M tiles per swizzle group
):
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
@@ -268,13 +268,13 @@ def _kernels() -> Optional[types.SimpleNamespace]:
N,
K,
stride_cm,
- HAS_BIAS: tl.constexpr,
- XS_FP32: tl.constexpr,
- WS_FP32: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- GROUP_M: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ XS_FP32: tl.constexpr, # activation scale is fp32
+ WS_FP32: tl.constexpr, # weight scale is fp32
+ BLOCK_M: tl.constexpr, # block size along M (rows)
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ GROUP_M: tl.constexpr, # M tiles per swizzle group
):
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
diff --git a/studio/backend/core/inference/diffusion_int8_gemv.py b/studio/backend/core/inference/diffusion_int8_gemv.py
index 985d84ce663..ae69bedbf03 100644
--- a/studio/backend/core/inference/diffusion_int8_gemv.py
+++ b/studio/backend/core/inference/diffusion_int8_gemv.py
@@ -64,12 +64,12 @@ def _kernel() -> Any:
stride_sn,
stride_on,
stride_ym,
- HAS_BIAS: tl.constexpr,
- HAS_OUT: tl.constexpr,
- GROUPED: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
+ HAS_BIAS: tl.constexpr, # bias present
+ HAS_OUT: tl.constexpr, # bf16 outlier columns present
+ GROUPED: tl.constexpr, # per-group weight scales
+ BLOCK_M: tl.constexpr, # block size along M (rows)
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
):
pid = tl.program_id(0)
rn = pid * BLOCK_N + tl.arange(0, BLOCK_N)
diff --git a/studio/backend/core/inference/diffusion_moss_fused.py b/studio/backend/core/inference/diffusion_moss_fused.py
index 0ef8633c155..f1ca2795bec 100644
--- a/studio/backend/core/inference/diffusion_moss_fused.py
+++ b/studio/backend/core/inference/diffusion_moss_fused.py
@@ -174,8 +174,8 @@ def _qk_rope_kernel() -> Any:
c2,
s2,
mask,
- EPS: tl.constexpr,
- D: tl.constexpr,
+ EPS: tl.constexpr, # norm epsilon, per model
+ D: tl.constexpr, # hd
):
# rotate(x) = [-x2, x1]; each product and sum rounded to bf16 like eager
x1 = tl.load(x_ptr + base + h * D + j, mask = mask, other = 0.0).to(tl.float32)
@@ -211,9 +211,9 @@ def _qk_rope_kernel() -> Any:
svb,
svs,
scs,
- EPS: tl.constexpr,
- D: tl.constexpr,
- HB: tl.constexpr,
+ EPS: tl.constexpr, # norm epsilon, per model
+ D: tl.constexpr, # hd
+ HB: tl.constexpr, # heads per program
):
row = tl.program_id(0)
b = row // S
diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
index cd229fe1ca3..1bb254142a2 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
@@ -50,10 +50,10 @@ if triton is not None:
q_ptr,
sf_ptr,
M,
- K: tl.constexpr,
- NCB: tl.constexpr,
- ROWS: tl.constexpr,
- G: tl.constexpr,
+ K: tl.constexpr, # in features
+ NCB: tl.constexpr, # scale tile columns (K/16/4)
+ ROWS: tl.constexpr, # rows per program
+ G: tl.constexpr, # 16-element blocks per program
):
# One program: ROWS rows x G blocks of 16 elements, held as [ROWS, G, 8] even / odd halves so each packed
# byte is (even | odd << 4) with no reshape or split (layout conversions some tile shapes miscompiled).
diff --git a/studio/backend/core/inference/diffusion_nvfp4_bias.py b/studio/backend/core/inference/diffusion_nvfp4_bias.py
index 52900b0fdef..a8b0aa35235 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_bias.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_bias.py
@@ -29,7 +29,9 @@ _FAST_BIAS_MAX_NUMEL = 2**31 - 1
if _HAVE_TRITON:
@triton.jit
- def _bias_add_kernel(ptr, bias_ptr, n_elements, n_cols: "tl.constexpr", BLOCK: "tl.constexpr"):
+ def _bias_add_kernel(
+ ptr, bias_ptr, n_elements, n_cols: "tl.constexpr", BLOCK: "tl.constexpr"
+ ): # n_cols out features, BLOCK block size
pid = tl.program_id(0)
offsets = pid * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_elements
diff --git a/studio/backend/core/inference/diffusion_qwenimage21_eager.py b/studio/backend/core/inference/diffusion_qwenimage21_eager.py
index 1dd62aea674..43094ef5a56 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21_eager.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21_eager.py
@@ -101,10 +101,10 @@ def _kernel() -> Any:
stride_h,
f_stride,
eps,
- HALF: tl.constexpr,
- ROWS: tl.constexpr,
- REAL_FMA_X: tl.constexpr,
- IMAG_FMA_X: tl.constexpr,
+ HALF: tl.constexpr, # hd // 2
+ ROWS: tl.constexpr, # rows per program
+ REAL_FMA_X: tl.constexpr, # fma pairing of real part
+ IMAG_FMA_X: tl.constexpr, # fma pairing of imag part
):
pid = tl.program_id(0).to(tl.int64)
rows = pid * ROWS + tl.arange(0, ROWS).to(tl.int64)
diff --git a/studio/backend/core/inference/diffusion_rocm_fused.py b/studio/backend/core/inference/diffusion_rocm_fused.py
index 5f5b6c44e52..efafd4bf875 100644
--- a/studio/backend/core/inference/diffusion_rocm_fused.py
+++ b/studio/backend/core/inference/diffusion_rocm_fused.py
@@ -239,7 +239,20 @@ def _adaln_kernel() -> Optional[Callable]:
@triton.jit
def _adaln_fwd(
- X, SCALE, SHIFT, OUT, S, D, sxb, sxs, scb, shb, sob, sos, eps, BLOCK: tl.constexpr
+ X,
+ SCALE,
+ SHIFT,
+ OUT,
+ S,
+ D,
+ sxb,
+ sxs,
+ scb,
+ shb,
+ sob,
+ sos,
+ eps,
+ BLOCK: tl.constexpr, # block size along hidden
):
row = tl.program_id(0)
b = row // S
diff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.py
index fb7ba9bdbc0..6d25a121270 100644
--- a/studio/backend/core/inference/diffusion_vae_fused.py
+++ b/studio/backend/core/inference/diffusion_vae_fused.py
@@ -71,7 +71,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
def _gn_partials(
x_ptr, bias_ptr, pn_ptr, pmean_ptr, pm2_ptr,
T, HW, W, n_chunks, sb, sc, st, sh, sw,
- C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr
+ C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr # C channels, G groupnorm groups, CPG channels per group, BLOCK_P pixels per program, HAS_BIAS bias present
): # fmt: skip
chunk = tl.program_id(0)
bt = tl.program_id(1)
@@ -106,7 +106,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit
def _gn_combine(
- pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr
+ pn_ptr,
+ pmean_ptr,
+ pm2_ptr,
+ mean_ptr,
+ rstd_ptr,
+ n_chunks,
+ G,
+ eps,
+ BLOCK: tl.constexpr, # block size
):
row = tl.program_id(0)
g = row % G
@@ -138,7 +146,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
def _gn_apply(
x_ptr, out_ptr, mean_ptr, rstd_ptr, w_ptr, b_ptr, ib_ptr,
C, T, HW, W, G, sb, sc, st, sh, sw,
- CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # CPG channels per group, ACT apply silu, HAS_IN_BIAS input bias present, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
@@ -179,8 +187,8 @@ def _kernels() -> Optional[types.SimpleNamespace]:
x_ptr, out_ptr, w_ptr, b_ptr, ib_ptr, cache_ptr,
C, T, H, W, Ho, Wo, ph, pw, To, front, n_cache, scale, eps,
sb, sc, st, sh, sw, cb, cc, ct, ch, cw,
- NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr,
- REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr, # NORM apply rms norm, ACT apply silu, HAS_BIAS bias present, HAS_IN_BIAS input bias present
+ REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # REPLICATE replicate temporal padding, MEAN_SQ mean-square rms form, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
@@ -252,7 +260,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit(do_not_specialize = _SHAPE_ARGS["_bias_residual"])
def _bias_residual(
o_ptr, ob_ptr, r_ptr, rb_ptr, P, C, T, HW, W, sb, sc, st, sh, sw, inv_scale,
- HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # HAS_OB output bias present, HAS_RB residual bias present, SCALE output scale != 1, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)
c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
@@ -285,7 +293,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
h_ptr, hb_ptr, x_ptr, out_ptr,
c, Cin, F, H, W, Fo, rep,
hsb, hsc, hst, hsh, hsw, xsb, xsc, xst, xsh, xsw,
- TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # TEMPORAL temporal upsample, HAS_HB conv bias present, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bf = tl.program_id(1)
@@ -338,7 +346,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
tl.store(out_ptr + o_off, (v + xv).to(out_ptr.dtype.element_ty), mask = m)
@triton.jit
- def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr):
+ def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr): # block size
row = tl.program_id(0).to(tl.int64)
base = s_ptr + row * P_STRIDE
m = tl.full([BLOCK], float("-inf"), tl.float32)
@@ -365,7 +373,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
def _dup_up_add(
o_ptr, x_ptr, C, T_o, H_o, W_o, t_off, repeats,
osb, osc, ost, osh, osw, xsb, xsc, xst, xsh, xsw,
- FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # FT temporal upsample factor, FS spatial upsample factor, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
@@ -399,7 +407,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit(do_not_specialize = _SHAPE_ARGS["_up_nearest2x"])
def _up_nearest2x(
x_ptr, out_ptr, C, T_o, H, W, xsb, xsc, xst, xsh, xsw,
- INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # INTERLEAVE wan time_conv channel interleave, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
# INTERLEAVE: x is Wan time_conv output (B, 2C, T_o/2, H, W); frame 2t+j takes channels [jC, (j+1)C).
pid_p = tl.program_id(0)
diff --git a/studio/backend/core/inference/video_minimax_h3_qknorm.py b/studio/backend/core/inference/video_minimax_h3_qknorm.py
index 03ffac4da86..0b39f02c640 100644
--- a/studio/backend/core/inference/video_minimax_h3_qknorm.py
+++ b/studio/backend/core/inference/video_minimax_h3_qknorm.py
@@ -64,9 +64,9 @@ def _kernel() -> Any:
stride_h,
cos_stride,
eps,
- D: tl.constexpr,
- HALF: tl.constexpr,
- ROWS: tl.constexpr,
+ D: tl.constexpr, # hd
+ HALF: tl.constexpr, # rotary dim // 2
+ ROWS: tl.constexpr, # rows per program
):
pid = tl.program_id(0).to(tl.int64)
rows = pid * ROWS + tl.arange(0, ROWS).to(tl.int64)
diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.py
index c48f3f1a4ad..d44a778ab7b 100644
--- a/studio/backend/core/inference/video_minimax_h3_vae.py
+++ b/studio/backend/core/inference/video_minimax_h3_vae.py
@@ -102,11 +102,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
st,
sh,
sw,
- C: tl.constexpr,
- G: tl.constexpr,
- CPG: tl.constexpr,
- BLOCK_P: tl.constexpr,
- HAS_BIAS: tl.constexpr,
+ C: tl.constexpr, # channels
+ G: tl.constexpr, # groupnorm groups
+ CPG: tl.constexpr, # channels per group
+ BLOCK_P: tl.constexpr, # pixels per program
+ HAS_BIAS: tl.constexpr, # bias present
):
chunk = tl.program_id(0)
bt = tl.program_id(1)
@@ -141,7 +141,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit
def _gn_combine(
- pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr
+ pn_ptr,
+ pmean_ptr,
+ pm2_ptr,
+ mean_ptr,
+ rstd_ptr,
+ n_chunks,
+ G,
+ eps,
+ BLOCK: tl.constexpr, # block size
):
row = tl.program_id(0)
g = row % G
@@ -194,11 +202,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
st,
sh,
sw,
- CPG: tl.constexpr,
- HAS_NORM: tl.constexpr,
- HAS_IN_BIAS: tl.constexpr,
- BLOCK_W: tl.constexpr,
- BLOCK_C: tl.constexpr,
+ CPG: tl.constexpr, # channels per group
+ HAS_NORM: tl.constexpr, # apply groupnorm
+ HAS_IN_BIAS: tl.constexpr, # input bias present
+ BLOCK_W: tl.constexpr, # width pixels per program
+ BLOCK_C: tl.constexpr, # channels per program
):
pid_row = tl.program_id(0)
pid_w = tl.program_id(1)
@@ -255,11 +263,11 @@ def _kernels() -> Optional[types.SimpleNamespace]:
st,
sh,
sw,
- C: tl.constexpr,
- HAS_OB: tl.constexpr,
- HAS_RB: tl.constexpr,
- R_SAME: tl.constexpr,
- BLOCK_P: tl.constexpr,
+ C: tl.constexpr, # channels
+ HAS_OB: tl.constexpr, # output bias present
+ HAS_RB: tl.constexpr, # residual bias present
+ R_SAME: tl.constexpr, # residual same layout (channels_last)
+ BLOCK_P: tl.constexpr, # pixels per program
):
p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)
pmask = p < P
@@ -294,7 +302,15 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit
def _add_rmsnorm(
- h_ptr, o_ptr, s_ptr, w_ptr, n_ptr, N, eps, HAS_RES: tl.constexpr, BLOCK_N: tl.constexpr
+ h_ptr,
+ o_ptr,
+ s_ptr,
+ w_ptr,
+ n_ptr,
+ N,
+ eps,
+ HAS_RES: tl.constexpr,
+ BLOCK_N: tl.constexpr, # HAS_RES residual add present, BLOCK_N block size along the row
):
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_N)
@@ -320,9 +336,9 @@ def _kernels() -> Optional[types.SimpleNamespace]:
heads,
cs_stride,
eps,
- HEAD_D: tl.constexpr,
- ROT: tl.constexpr,
- BLOCK: tl.constexpr,
+ HEAD_D: tl.constexpr, # hd
+ ROT: tl.constexpr, # rotary dim
+ BLOCK: tl.constexpr, # block size
):
hr = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
rmask = hr < n_head_rows
@@ -350,7 +366,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
tl.store(x_ptr + base + d[None, :], y.to(dt), mask = rmask[:, None])
@triton.jit
- def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr):
+ def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr): # block size along the row
row = tl.program_id(0).to(tl.int64)
pid_n = tl.program_id(1)
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
@@ -362,7 +378,7 @@ def _kernels() -> Optional[types.SimpleNamespace]:
tl.store(out_ptr + row * N + cols, (hid * act).to(dt), mask = mask)
@triton.jit
- def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr):
+ def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr): # block size along the row
# IEEE division + half-to-even rounding: the same codes as the torch path.
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_N)
@@ -377,7 +393,14 @@ def _kernels() -> Optional[types.SimpleNamespace]:
@triton.jit
def _dequant_epilogue(
- acc_ptr, xs_ptr, ws_ptr, bias_ptr, out_ptr, N, HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr
+ acc_ptr,
+ xs_ptr,
+ ws_ptr,
+ bias_ptr,
+ out_ptr,
+ N,
+ HAS_BIAS: tl.constexpr,
+ BLOCK_N: tl.constexpr, # HAS_BIAS bias present, BLOCK_N block size along the row
):
row = tl.program_id(0).to(tl.int64)
pid_n = tl.program_id(1)
diff --git a/studio/backend/core/inference/video_wan_fused.py b/studio/backend/core/inference/video_wan_fused.py
index 8e73e62f3a1..c15110a59c0 100644
--- a/studio/backend/core/inference/video_wan_fused.py
+++ b/studio/backend/core/inference/video_wan_fused.py
@@ -83,9 +83,9 @@ def _kernels() -> Optional[dict]:
stl,
stk,
sbk,
- K_SHIFT: tl.constexpr,
- K_SCALE: tl.constexpr,
- BLOCK: tl.constexpr,
+ K_SHIFT: tl.constexpr, # modulation index of shift
+ K_SCALE: tl.constexpr, # modulation index of scale
+ BLOCK: tl.constexpr, # block size along hidden
):
# int64: l * 6 * D passes 2**31 on long clips; int32 wraps to an out-of-bounds temb read.
row = tl.program_id(0).to(tl.int64)
@@ -110,7 +110,19 @@ def _kernels() -> Optional[dict]:
@triton.jit
def _gate_residual(
- X, A, T, TBL, OUT, D, L, stb, stl, stk, sbk, K_GATE: tl.constexpr, BLOCK: tl.constexpr
+ X,
+ A,
+ T,
+ TBL,
+ OUT,
+ D,
+ L,
+ stb,
+ stl,
+ stk,
+ sbk,
+ K_GATE: tl.constexpr,
+ BLOCK: tl.constexpr, # K_GATE modulation index of gate, BLOCK block size along hidden
):
# out = fp16(x.float() + a.float() * (tbl[gate] + temb[gate]))
row = tl.program_id(0).to(tl.int64) # int64 temb offset, as in _modnorm
@@ -130,7 +142,9 @@ def _kernels() -> Optional[dict]:
tl.store(OUT + row * D + col, y.to(OUT.dtype.element_ty), mask = mask)
@triton.jit
- def _affine_norm(X, MEAN, RSTD, W, BIAS, OUT, D, BLOCK: tl.constexpr):
+ def _affine_norm(
+ X, MEAN, RSTD, W, BIAS, OUT, D, BLOCK: tl.constexpr
+ ): # block size along hidden
# Matches PyTorch's gamma * (...) + beta, which nvcc contracts to one fma.
row = tl.program_id(0)
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
@@ -145,7 +159,16 @@ def _kernels() -> Optional[dict]:
@triton.jit
def _rope(
- X, COS, SIN, OUT, L, scl, ssl, PAIRS: tl.constexpr, HALF: tl.constexpr, BLOCK: tl.constexpr
+ X,
+ COS,
+ SIN,
+ OUT,
+ L,
+ scl,
+ ssl,
+ PAIRS: tl.constexpr,
+ HALF: tl.constexpr,
+ BLOCK: tl.constexpr, # PAIRS rotary pairs per row, HALF hd // 2, BLOCK block size along hidden
):
# Each product a separately rounded fp32 multiply, then one fp16 rounding, matching stock.
row = tl.program_id(0)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 087a758ec21..af1b54b4a77 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -59,12 +59,12 @@ def _cross_entropy_forward(
loss_ptr,
logsumexp_ptr,
labels_ptr,
- VOCAB_SIZE: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- DO_SOFTCAPPING: tl.constexpr,
- SOFTCAP: tl.constexpr,
- DO_LOGIT_SCALING: tl.constexpr,
- LOGIT_SCALE: tl.constexpr,
+ VOCAB_SIZE: tl.constexpr, # vocab size
+ BLOCK_SIZE: tl.constexpr, # vocab block size
+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on
+ SOFTCAP: tl.constexpr, # softcap value
+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on
+ LOGIT_SCALE: tl.constexpr, # logit scale value
):
"""
Cross Entropy Loss = 1/n sum [ -yi log(Pi) ]
@@ -186,19 +186,19 @@ def _chunked_forward_config(device, vocab_size):
def _chunked_cross_entropy_forward(
logits_ptr,
- logits_row_stride: tl.constexpr,
+ logits_row_stride: tl.constexpr, # logits row stride (vocab)
loss_ptr,
logsumexp_ptr,
logsumexp_others_ptr,
labels_ptr,
- VOCAB_SIZE: tl.constexpr,
- N_CHUNKS: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- DO_SOFTCAPPING: tl.constexpr,
- SOFTCAP: tl.constexpr,
- DO_LOGIT_SCALING: tl.constexpr,
- LOGIT_SCALE: tl.constexpr,
- EVICT: tl.constexpr,
+ VOCAB_SIZE: tl.constexpr, # vocab size
+ N_CHUNKS: tl.constexpr, # vocab chunks per row
+ BLOCK_SIZE: tl.constexpr, # vocab block size
+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on
+ SOFTCAP: tl.constexpr, # softcap value
+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on
+ LOGIT_SCALE: tl.constexpr, # logit scale value
+ EVICT: tl.constexpr, # logits load eviction policy
):
"""
256K vocab divided in 4 chunks
@@ -298,19 +298,19 @@ _chunked_cross_entropy_forward = triton.heuristics(
def _cross_entropy_backward(
logits_ptr,
- logits_row_stride: tl.constexpr,
+ logits_row_stride: tl.constexpr, # logits row stride (vocab)
dlogits_ptr,
dloss_ptr,
- dloss_row_stride: tl.constexpr,
+ dloss_row_stride: tl.constexpr, # dloss row stride
logsumexp_ptr,
loss_ptr,
labels_ptr,
- VOCAB_SIZE: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- DO_SOFTCAPPING: tl.constexpr,
- SOFTCAP: tl.constexpr,
- DO_LOGIT_SCALING: tl.constexpr,
- LOGIT_SCALE: tl.constexpr,
+ VOCAB_SIZE: tl.constexpr, # vocab size
+ BLOCK_SIZE: tl.constexpr, # vocab block size
+ DO_SOFTCAPPING: tl.constexpr, # logit softcapping on
+ SOFTCAP: tl.constexpr, # softcap value
+ DO_LOGIT_SCALING: tl.constexpr, # logit scaling on
+ LOGIT_SCALE: tl.constexpr, # logit scale value
):
"""
CE_i = -y log(P) = y * (log[sum(exp(x))] - x)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 39e591a81f1..00593ffefd4 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -197,8 +197,8 @@ def weight_dequant_kernel(
y_ptr,
M,
N,
- BLOCK_SIZE: tl.constexpr,
- EXACT_F16: tl.constexpr = False,
+ BLOCK_SIZE: tl.constexpr, # fp8 quant block size
+ EXACT_F16: tl.constexpr = False, # exact fp16 rounding of output
):
pid_m = tl.program_id(axis = 0)
pid_n = tl.program_id(axis = 1)
@@ -381,8 +381,8 @@ def act_quant_kernel(
x_ptr,
y_ptr,
s_ptr,
- BLOCK_SIZE: tl.constexpr,
- LONG_INDEXING: tl.constexpr = False,
+ BLOCK_SIZE: tl.constexpr, # fp8 quant block size
+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31
):
pid = tl.program_id(axis = 0)
if LONG_INDEXING:
@@ -448,11 +448,11 @@ def _w8a8_block_fp8_matmul(
stride_As_k,
stride_Bs_k,
stride_Bs_n,
- BLOCK_SIZE_M: tl.constexpr,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- GROUP_SIZE_M: tl.constexpr,
- LONG_INDEXING: tl.constexpr = False,
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ GROUP_SIZE_M: tl.constexpr, # M tiles per swizzle group
+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31
):
"""Triton-accelerated function used to perform linear operations (dot
product) on input tensors `A` and `B` with block-wise quantization, and
@@ -542,12 +542,12 @@ def _w8a8_block_fp8_matmul_tma(
stride_As_k,
stride_Bs_k,
stride_Bs_n,
- BLOCK_SIZE_M: tl.constexpr,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- GROUP_SIZE_M: tl.constexpr,
- NUM_STAGES: tl.constexpr,
- OUT_DTYPE: tl.constexpr,
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ GROUP_SIZE_M: tl.constexpr, # M tiles per swizzle group
+ NUM_STAGES: tl.constexpr, # pipeline stages
+ OUT_DTYPE: tl.constexpr, # output dtype
):
# _w8a8_block_fp8_matmul with A, B and C moved by TMA descriptors (out-of-range rows and K tail
# are zero filled on load, clipped on store) and a pipelined K loop; same per-tile math and order,
@@ -851,11 +851,11 @@ def _w8a8_block_fp8_small_m(
stride_Bs_n,
stride_Bs_k,
KB_PER_SPLIT,
- GROUP_N: tl.constexpr,
- BLOCK_SIZE_M: tl.constexpr,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- SPLIT_K: tl.constexpr,
+ GROUP_N: tl.constexpr, # weight scale block along N
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ SPLIT_K: tl.constexpr, # split-K factor
):
pid_n = tl.program_id(axis = 0)
pid_k = tl.program_id(axis = 1)
@@ -1208,9 +1208,9 @@ def _fp8_rowwise_gemv_kernel(
N,
K,
stride_wn,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- LONG_INDEXING: tl.constexpr = False,
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31
):
pid = tl.program_id(0)
# Widen before the multiply: row * stride_wn overflows int32 past 2^31 weight elements.
@@ -1254,11 +1254,11 @@ def _fp8_rowwise_gemv_rows_kernel(
N,
K,
stride_wn,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- SPLIT_K: tl.constexpr,
- LONG_INDEXING: tl.constexpr = False,
+ BLOCK_M: tl.constexpr, # rows per program
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ SPLIT_K: tl.constexpr, # split-K factor
+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31
):
pid = tl.program_id(0)
pid_k = tl.program_id(1)
@@ -1297,7 +1297,13 @@ def _fp8_rowwise_gemv_rows_kernel(
@triton.jit(do_not_specialize = ["M"])
def _fp8_rowwise_gemv_rows_reduce(
- p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr
+ p_ptr,
+ s_ptr,
+ y_ptr,
+ M,
+ N,
+ SPLIT_K: tl.constexpr,
+ BLOCK: tl.constexpr, # SPLIT_K split-K factor, BLOCK block size
):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < M * N
@@ -1453,12 +1459,12 @@ def _fp8_block_gemv_kernel(
stride_sk,
stride_ys,
stride_ym,
- GROUP_N: tl.constexpr,
- GROUP_K: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- LONG_INDEXING: tl.constexpr = False,
+ GROUP_N: tl.constexpr, # weight scale block along N
+ GROUP_K: tl.constexpr, # weight scale block along K
+ BLOCK_M: tl.constexpr, # rows per program
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ LONG_INDEXING: tl.constexpr = False, # int64 offsets past 2**31
):
pid_n = tl.program_id(0)
pid_k = tl.program_id(1)
@@ -1713,12 +1719,12 @@ def _e4m3fn_bytes_dequant_kernel(
stride_xn,
stride_sm,
stride_sn,
- BLOCK_M_Q: tl.constexpr,
- BLOCK_N_Q: tl.constexpr,
- BLOCK_R: tl.constexpr,
- BLOCK_C: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- EXACT_F16: tl.constexpr = False,
+ BLOCK_M_Q: tl.constexpr, # scale block rows
+ BLOCK_N_Q: tl.constexpr, # scale block cols
+ BLOCK_R: tl.constexpr, # rows per program
+ BLOCK_C: tl.constexpr, # cols per program
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ EXACT_F16: tl.constexpr = False, # exact fp16 rounding of output
):
pid_r = tl.program_id(0)
pid_c = tl.program_id(1)
@@ -1754,11 +1760,11 @@ def _e4m3fn_scale_dequant_kernel(
M,
N,
stride_s,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- SCALE_ON_COLS: tl.constexpr,
- ROUND_SCALE: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
+ BLOCK_M: tl.constexpr, # rows per program
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ SCALE_ON_COLS: tl.constexpr, # scale per column (transposed)
+ ROUND_SCALE: tl.constexpr, # round scale to output dtype
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
):
# Scale per tensor (stride_s 0), row or column. ROUND_SCALE mirrors x.to(dtype) * s.to(dtype) (fp16 keeps
# the fp32 scale); e4m3 times a bf16 scale is exact in fp32, so one rounding gives torch's bits.
@@ -2233,9 +2239,9 @@ def _fp8_act_quant_per_row_kernel(
ub_ptr,
K,
stride_xm,
- HAS_UB: tl.constexpr,
- BLOCK_K: tl.constexpr,
- ONE_BLOCK: tl.constexpr,
+ HAS_UB: tl.constexpr, # scale upper bound present
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ ONE_BLOCK: tl.constexpr, # row fits one BLOCK_K
):
# Same fp32 math Inductor emits for _quantize_fp8_per_row; [1, BLOCK_K] tile is faster than 1D on sm120.
row = tl.program_id(0).to(tl.int64)
@@ -2277,7 +2283,7 @@ def _fp8_act_quant_per_row_kernel(
@triton.jit
-def _fp8_rowwise_dequant_kernel(w_ptr, s_ptr, y_ptr, numel, K, BLOCK: tl.constexpr):
+def _fp8_rowwise_dequant_kernel(w_ptr, s_ptr, y_ptr, numel, K, BLOCK: tl.constexpr): # block size
# One fp32 multiply, one rounding, as Inductor fuses it (eager x.to(dtype) * s.to(dtype) rounds twice).
offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
mask = offs < numel
diff --git a/unsloth/kernels/fused_add_rms_layernorm.py b/unsloth/kernels/fused_add_rms_layernorm.py
index c0a23a8d441..98423cb1690 100644
--- a/unsloth/kernels/fused_add_rms_layernorm.py
+++ b/unsloth/kernels/fused_add_rms_layernorm.py
@@ -25,10 +25,10 @@ def _add_rms_layernorm_inference(
W,
X_row_stride,
R_row_stride,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- HAS_RESIDUAL: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
+ n_cols: tl.constexpr, # hidden size
+ eps: tl.constexpr, # norm epsilon, per model
+ HAS_RESIDUAL: tl.constexpr, # add residual before norm
+ BLOCK_SIZE: tl.constexpr, # block size, next pow2 of row width
):
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_SIZE)
diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.py
index e6884506ac2..992ec705b30 100644
--- a/unsloth/kernels/geglu.py
+++ b/unsloth/kernels/geglu.py
@@ -27,7 +27,12 @@ INT32_SAFETY_BUFFER = NUM_INT32_ELEMENTS - BLOCK_SIZE * SAFE_INT32_BUFFER_MULTIP
@triton.jit
def _exact_forward_kernel(
- e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+ e,
+ g,
+ h,
+ n_elements,
+ BLOCK_SIZE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
):
block_idx = tl.program_id(0)
if LONG_INDEXING:
@@ -69,7 +74,12 @@ def geglu_exact_forward_kernel(gate, up):
@triton.jit
def _exact_backward_kernel(
- DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+ DW,
+ e,
+ g,
+ n_elements,
+ BLOCK_SIZE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
):
"""
f = 1/2 * e * (1 + erf(1/sqrt(2) * e))
@@ -136,7 +146,12 @@ def geglu_exact_backward_kernel(DW, e, g):
@triton.jit
def _approx_forward_kernel(
- e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+ e,
+ g,
+ h,
+ n_elements,
+ BLOCK_SIZE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
):
block_idx = tl.program_id(0)
if LONG_INDEXING:
@@ -180,7 +195,12 @@ def geglu_approx_forward_kernel(gate, up):
@triton.jit
def _approx_backward_kernel(
- DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+ DW,
+ e,
+ g,
+ n_elements,
+ BLOCK_SIZE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr, # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
):
"""
f = 1/2 * e * (1 + tanh( sqrt(2/pi) * x * (1 + 0.044715 * x^2 ) ))
diff --git a/unsloth/kernels/int4_packed.py b/unsloth/kernels/int4_packed.py
index fefae25ab0c..e99a6029fb5 100644
--- a/unsloth/kernels/int4_packed.py
+++ b/unsloth/kernels/int4_packed.py
@@ -88,11 +88,11 @@ def _unpack_tile3(
stride_pn,
stride_sn,
stride_zn,
- BITS: tl.constexpr,
- PF: tl.constexpr,
- GROUP: tl.constexpr,
- HAS_Z: tl.constexpr,
- HAS_G: tl.constexpr,
+ BITS: tl.constexpr, # bits per weight (4 or 8)
+ PF: tl.constexpr, # values packed per int32 word
+ GROUP: tl.constexpr, # quant group size along K
+ HAS_Z: tl.constexpr, # zero points present
+ HAS_G: tl.constexpr, # act-order g_idx present
):
# fp32 decode of W[rn, words rkp] as [BLOCK_N, BLOCK_KP, PF] (column = word * PF + j).
MASK: tl.constexpr = (1 << BITS) - 1
@@ -150,13 +150,13 @@ def _unpack_tile(
stride_pn,
stride_sn,
stride_zn,
- BITS: tl.constexpr,
- PF: tl.constexpr,
- GROUP: tl.constexpr,
- HAS_Z: tl.constexpr,
- HAS_G: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_KP: tl.constexpr,
+ BITS: tl.constexpr, # bits per weight (4 or 8)
+ PF: tl.constexpr, # values packed per int32 word
+ GROUP: tl.constexpr, # quant group size along K
+ HAS_Z: tl.constexpr, # zero points present
+ HAS_G: tl.constexpr, # act-order g_idx present
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_KP: tl.constexpr, # packed words per program along K
):
w = _unpack_tile3(
P, S, Z, G, rn, rkp, N, K, KP, stride_pn, stride_sn, stride_zn,
@@ -179,13 +179,13 @@ def _dequant_kernel(
stride_sn,
stride_zn,
stride_on,
- BITS: tl.constexpr,
- PF: tl.constexpr,
- GROUP: tl.constexpr,
- HAS_Z: tl.constexpr,
- HAS_G: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_KP: tl.constexpr,
+ BITS: tl.constexpr, # bits per weight (4 or 8)
+ PF: tl.constexpr, # values packed per int32 word
+ GROUP: tl.constexpr, # quant group size along K
+ HAS_Z: tl.constexpr, # zero points present
+ HAS_G: tl.constexpr, # act-order g_idx present
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_KP: tl.constexpr, # packed words per program along K
):
pid_n = tl.program_id(0)
pid_k = tl.program_id(1)
@@ -221,14 +221,14 @@ def _gemv_kernel(
stride_sn,
stride_zn,
stride_ym,
- BITS: tl.constexpr,
- PF: tl.constexpr,
- GROUP: tl.constexpr,
- HAS_Z: tl.constexpr,
- HAS_G: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_KP: tl.constexpr,
- SPLIT_K: tl.constexpr,
+ BITS: tl.constexpr, # bits per weight (4 or 8)
+ PF: tl.constexpr, # values packed per int32 word
+ GROUP: tl.constexpr, # quant group size along K
+ HAS_Z: tl.constexpr, # zero points present
+ HAS_G: tl.constexpr, # act-order g_idx present
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_KP: tl.constexpr, # packed words per program along K
+ SPLIT_K: tl.constexpr, # split-K factor
):
# Decode: one activation row per program axis 2 (M tiny), CUDA-core reduction.
pid_n = tl.program_id(0)
@@ -565,13 +565,13 @@ def _dequant_repacked_kernel(
stride_sn,
stride_zn,
stride_on,
- MARLIN: tl.constexpr,
- TN: tl.constexpr,
- TK: tl.constexpr,
- R: tl.constexpr,
- GROUP: tl.constexpr,
- HAS_Z: tl.constexpr,
- RAW: tl.constexpr,
+ MARLIN: tl.constexpr, # marlin tile layout (else tinygemm)
+ TN: tl.constexpr, # tile rows (N)
+ TK: tl.constexpr, # tile cols (K)
+ R: tl.constexpr, # tiles along K per program
+ GROUP: tl.constexpr, # quant group size along K
+ HAS_Z: tl.constexpr, # zero points present
+ RAW: tl.constexpr, # store raw int codes
):
# One program: tile row tn, R tiles along K, written as [TN, R * TK] contiguous rows.
pid = tl.program_id(0)
diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.py
index f39638b788e..a114dd2cf3b 100644
--- a/unsloth/kernels/large_head_attention.py
+++ b/unsloth/kernels/large_head_attention.py
@@ -78,9 +78,9 @@ def _lh_fwd(
H,
G,
S,
- HEAD_DIM: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_M: tl.constexpr, # query rows per program
+ BLOCK_N: tl.constexpr, # key rows per step
):
start_m = tl.program_id(0)
off_bh = tl.program_id(1)
@@ -147,8 +147,8 @@ def _lh_bwd_pre(
sdom,
H,
S,
- HEAD_DIM: tl.constexpr,
- BLOCK_M: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_M: tl.constexpr, # query rows per program
):
start_m = tl.program_id(0)
off_bh = tl.program_id(1)
@@ -203,9 +203,9 @@ def _lh_bwd_dq(
H,
G,
S,
- HEAD_DIM: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_M: tl.constexpr, # query rows per program
+ BLOCK_N: tl.constexpr, # key rows per step
):
start_m = tl.program_id(0)
off_bh = tl.program_id(1)
@@ -291,9 +291,9 @@ def _lh_bwd_dkdv(
HKV,
G,
S,
- HEAD_DIM: tl.constexpr,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_M: tl.constexpr, # query rows per program
+ BLOCK_N: tl.constexpr, # key rows per step
):
# One program owns a kv block of one kv head and loops over its G query heads,
# so GQA needs no atomics and no repeated K / V.
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 6942271b3b8..21768362cc2 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -32,11 +32,11 @@ def layernorm_forward(
mu,
Y_row_stride: tl.constexpr, # n_cols
X_row_stride: tl.constexpr, # n_cols
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- HAS_BIAS: tl.constexpr = True,
+ n_cols: tl.constexpr, # hidden size
+ eps: tl.constexpr, # norm epsilon, per model
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # hidden next pow2
+ HAS_BIAS: tl.constexpr = True, # bias present
):
row_idx = tl.program_id(0)
if LONG_INDEXING:
@@ -70,7 +70,16 @@ def layernorm_forward(
@triton.jit
def _layernorm_backward_row(
- dY, dX, X, W_row, r, mu, row_idx, col_offsets, mask, n_cols: tl.constexpr
+ dY,
+ dX,
+ X,
+ W_row,
+ r,
+ mu,
+ row_idx,
+ col_offsets,
+ mask,
+ n_cols: tl.constexpr, # hidden size
):
# based on karpathy/llm.c; float32 matches torchtune's Fp32LayerNorm.
dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)
@@ -95,13 +104,13 @@ def layernorm_backward(
b,
r,
mu,
- dY_row_stride: tl.constexpr,
- dX_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ n_cols: tl.constexpr, # hidden size
+ eps: tl.constexpr, # norm epsilon, per model
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # hidden next pow2
):
row_idx = tl.program_id(0)
if LONG_INDEXING:
@@ -136,14 +145,14 @@ def layernorm_backward_dw(
dB,
n_rows,
rows_per_program,
- dY_row_stride: tl.constexpr,
- dX_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- TRAIN_W: tl.constexpr,
- TRAIN_B: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ n_cols: tl.constexpr, # hidden size
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # hidden next pow2
+ TRAIN_W: tl.constexpr, # weight grad partials
+ TRAIN_B: tl.constexpr, # bias grad partials
):
pid = tl.program_id(0)
col_offsets = tl.arange(0, BLOCK_SIZE)
@@ -180,7 +189,13 @@ def layernorm_backward_dw(
@triton.jit
def _layernorm_sum_partials(
- partials, out0, out1, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr
+ partials,
+ out0,
+ out1,
+ n_programs,
+ n_cols,
+ BLOCK_ROWS: tl.constexpr,
+ BLOCK_COLS: tl.constexpr, # BLOCK_ROWS partial rows per program, BLOCK_COLS columns per program
):
# Not torch.sum: Inductor reorders it, so eager and compiled bytes would differ.
which = tl.program_id(1)
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index 9e2cd97ba6b..5afa5e70753 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -51,21 +51,21 @@ def _grouped_gemm_dX_kernel(
dX_ptr,
gather_indices_ptr,
m_sizes_ptr,
- NUM_EXPERTS: tl.constexpr,
+ NUM_EXPERTS: tl.constexpr, # number of experts
NUM_TOKENS,
- TOPK: tl.constexpr,
- N: tl.constexpr,
- K: tl.constexpr,
+ TOPK: tl.constexpr, # experts per token (top-k)
+ N: tl.constexpr, # out features per expert
+ K: tl.constexpr, # in features per expert
NUM_SMS,
- BLOCK_SIZE_M: tl.constexpr,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- PERMUTE_X: tl.constexpr = False,
- PERMUTE_Y: tl.constexpr = False,
- USE_TMA_LOAD_W: tl.constexpr = False,
- USE_TMA_LOAD_dY: tl.constexpr = False,
- USE_TMA_STORE: tl.constexpr = False,
- FLATTEN: tl.constexpr = True,
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing
+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing
+ USE_TMA_LOAD_W: tl.constexpr = False, # TMA load for W
+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY
+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output
+ FLATTEN: tl.constexpr = True, # flatten the expert loop
) -> None:
TOTAL_TOKENS = NUM_TOKENS * TOPK
output_dtype = dX_ptr.dtype.element_ty
@@ -239,21 +239,21 @@ def _grouped_gemm_dW_kernel(
m_sizes_ptr,
gather_indices_ptr,
NUM_TOKENS,
- TOPK: tl.constexpr,
- NUM_EXPERTS: tl.constexpr,
- N: tl.constexpr,
- K: tl.constexpr,
+ TOPK: tl.constexpr, # experts per token (top-k)
+ NUM_EXPERTS: tl.constexpr, # number of experts
+ N: tl.constexpr, # out features per expert
+ K: tl.constexpr, # in features per expert
NUM_SMS,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- BLOCK_SIZE_M: tl.constexpr,
- PERMUTE_X: tl.constexpr = False,
- PERMUTE_Y: tl.constexpr = False,
- USE_TMA_LOAD_dY: tl.constexpr = False,
- USE_TMA_LOAD_X: tl.constexpr = False,
- USE_TMA_STORE: tl.constexpr = False,
- acc_dtype: tl.constexpr = tl.float32,
- ACCUMULATE: tl.constexpr = False,
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing
+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing
+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY
+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X
+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output
+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype
+ ACCUMULATE: tl.constexpr = False, # accumulate into existing dW
# Seed the fp32 accumulator with the existing tile (load overlaps the main loop) instead of adding
# it after the loop. Only for a bf16 / fp16 dW, whose final rounding dwarfs the fp32 order change;
# an fp32 dW adds at the end, the same single fp32 add autograd would do.
@@ -436,22 +436,22 @@ def _grouped_gemm_dW_per_expert_kernel(
m_sizes_ptr,
gather_indices_ptr,
NUM_TOKENS,
- TOPK: tl.constexpr,
- NUM_EXPERTS: tl.constexpr,
- N: tl.constexpr,
- K: tl.constexpr,
+ TOPK: tl.constexpr, # experts per token (top-k)
+ NUM_EXPERTS: tl.constexpr, # number of experts
+ N: tl.constexpr, # out features per expert
+ K: tl.constexpr, # in features per expert
NUM_SMS,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- BLOCK_SIZE_M: tl.constexpr,
- PERMUTE_X: tl.constexpr = False,
- PERMUTE_Y: tl.constexpr = False,
- USE_TMA_LOAD_dY: tl.constexpr = False,
- USE_TMA_LOAD_X: tl.constexpr = False,
- USE_TMA_STORE: tl.constexpr = False,
- acc_dtype: tl.constexpr = tl.float32,
- ACCUMULATE: tl.constexpr = False,
- EXPERTS_POW2: tl.constexpr = 1,
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing
+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing
+ USE_TMA_LOAD_dY: tl.constexpr = False, # TMA load for dY
+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X
+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output
+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype
+ ACCUMULATE: tl.constexpr = False, # accumulate into existing dW
+ EXPERTS_POW2: tl.constexpr = 1, # num experts next pow2
) -> None:
"""dW[e] = dY_e^T X_e with one program per (expert, n tile, k tile).
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
index 6960cb33d90..3c53a2c6abe 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
@@ -23,26 +23,26 @@ def _grouped_gemm_forward_kernel(
gather_indices_ptr,
topk_weights_ptr,
# Constant problem shapes
- NUM_EXPERTS: tl.constexpr,
+ NUM_EXPERTS: tl.constexpr, # number of experts
NUM_TOKENS,
- TOPK: tl.constexpr,
- N: tl.constexpr,
- K: tl.constexpr,
+ TOPK: tl.constexpr, # experts per token (top-k)
+ N: tl.constexpr, # out features per expert
+ K: tl.constexpr, # in features per expert
NUM_SMS,
# Tuning params
- BLOCK_SIZE_M: tl.constexpr,
- BLOCK_SIZE_N: tl.constexpr,
- BLOCK_SIZE_K: tl.constexpr,
- PERMUTE_X: tl.constexpr = False,
- PERMUTE_Y: tl.constexpr = False,
- FUSE_MUL_PRE: tl.constexpr = False,
- FUSE_MUL_POST: tl.constexpr = False,
- USE_FAST_ACCUM: tl.constexpr = False,
- USE_TMA_LOAD_W: tl.constexpr = False,
- USE_TMA_LOAD_X: tl.constexpr = False,
- USE_TMA_STORE: tl.constexpr = False,
- acc_dtype: tl.constexpr = tl.float32,
- FLATTEN: tl.constexpr = True,
+ BLOCK_SIZE_M: tl.constexpr, # block size along M (rows)
+ BLOCK_SIZE_N: tl.constexpr, # block size along N (out features)
+ BLOCK_SIZE_K: tl.constexpr, # block size along K (in features)
+ PERMUTE_X: tl.constexpr = False, # gather X rows by routing
+ PERMUTE_Y: tl.constexpr = False, # scatter Y rows by routing
+ FUSE_MUL_PRE: tl.constexpr = False, # multiply topk weight before gemm
+ FUSE_MUL_POST: tl.constexpr = False, # multiply topk weight after gemm
+ USE_FAST_ACCUM: tl.constexpr = False, # fp8 fast accumulation
+ USE_TMA_LOAD_W: tl.constexpr = False, # TMA load for W
+ USE_TMA_LOAD_X: tl.constexpr = False, # TMA load for X
+ USE_TMA_STORE: tl.constexpr = False, # TMA store for output
+ acc_dtype: tl.constexpr = tl.float32, # accumulator dtype
+ FLATTEN: tl.constexpr = True, # flatten the expert loop
) -> None:
tl.static_assert(K % BLOCK_SIZE_K == 0)
diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.py
index 507766e42ed..871d4c03d60 100644
--- a/unsloth/kernels/nvfp4.py
+++ b/unsloth/kernels/nvfp4.py
@@ -81,10 +81,10 @@ def _nvfp4_dequant_kernel(
rows,
half_cols,
scale_cols,
- BLOCK_R: tl.constexpr,
- BLOCK_C: tl.constexpr,
- SCALE_E4M3_BYTES: tl.constexpr,
- INT64_OFFSETS: tl.constexpr,
+ BLOCK_R: tl.constexpr, # rows per program
+ BLOCK_C: tl.constexpr, # cols per program
+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes
+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31
):
r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
if INT64_OFFSETS:
@@ -193,12 +193,12 @@ def _nvfp4_weight_tile(
stride_pc,
stride_sr,
stride_sc,
- BLOCK_R: tl.constexpr,
- BLOCK_K: tl.constexpr,
- SCALE_E4M3_BYTES: tl.constexpr,
- OUT_DTYPE: tl.constexpr,
- INT64_OFFSETS: tl.constexpr,
- MASK_PAD: tl.constexpr,
+ BLOCK_R: tl.constexpr, # rows per program
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes
+ OUT_DTYPE: tl.constexpr, # output dtype
+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31
+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)
):
# Rounded like dense: E2M1 * (block scale / global scale) in fp32, then one cast; out-of-range decodes to 0.
c = k0 // 2 + tl.arange(0, BLOCK_K // 2)
@@ -251,13 +251,13 @@ def _nvfp4_matmul_fwd_kernel(
stride_yn,
stride_ys,
k_per_split,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- GROUP_M: tl.constexpr,
- SCALE_E4M3_BYTES: tl.constexpr,
- INT64_OFFSETS: tl.constexpr,
- MASK_PAD: tl.constexpr,
+ BLOCK_M: tl.constexpr, # block size along M (rows)
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ GROUP_M: tl.constexpr, # M tiles per swizzle group
+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes
+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31
+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)
):
# Split K writes fp32 partials [split, M, N], summed in a fixed order afterwards (deterministic).
pid = tl.program_id(0)
@@ -339,13 +339,13 @@ def _nvfp4_matmul_dx_kernel(
stride_dk,
stride_ds,
n_per_split,
- BLOCK_M: tl.constexpr,
- BLOCK_N: tl.constexpr,
- BLOCK_K: tl.constexpr,
- GROUP_M: tl.constexpr,
- SCALE_E4M3_BYTES: tl.constexpr,
- INT64_OFFSETS: tl.constexpr,
- MASK_PAD: tl.constexpr,
+ BLOCK_M: tl.constexpr, # block size along M (rows)
+ BLOCK_N: tl.constexpr, # block size along N (out features)
+ BLOCK_K: tl.constexpr, # block size along K (in features)
+ GROUP_M: tl.constexpr, # M tiles per swizzle group
+ SCALE_E4M3_BYTES: tl.constexpr, # scales stored as raw e4m3 bytes
+ INT64_OFFSETS: tl.constexpr, # int64 offsets past 2**31
+ MASK_PAD: tl.constexpr, # mask partial tiles (N or K ragged)
):
pid = tl.program_id(0)
split_id = tl.program_id(1)
@@ -555,7 +555,9 @@ def _fused_config(
@triton.jit
-def _split_sum_kernel(partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr):
+def _split_sum_kernel(
+ partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr
+): # SPLIT split-K partials, BLOCK block size
# Fixed-order sum of the split partials (determinism), one cast.
offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
mask = offs < numel
diff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.py
index 60b0541ab01..44ba7d92dc2 100644
--- a/unsloth/kernels/qk_norm_rope.py
+++ b/unsloth/kernels/qk_norm_rope.py
@@ -56,8 +56,8 @@ def _norm_rope_fwd_block(
sin1,
cols,
eps,
- HEAD_DIM: tl.constexpr,
- HAS_ROPE_INDICES: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids
):
HALF = HEAD_DIM // 2
hmask = heads < n_heads
@@ -108,9 +108,9 @@ def _qk_norm_rope_forward(
sin_row_stride,
rope_indices,
seqlen,
- HEAD_DIM: tl.constexpr,
- BLOCK_H: tl.constexpr,
- HAS_ROPE_INDICES: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_H: tl.constexpr, # heads per program
+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids
):
row = tl.program_id(0).to(tl.int64)
hblock = tl.program_id(1)
@@ -156,7 +156,7 @@ def _norm_rope_bwd_block(
cos1,
sin1,
cols,
- HEAD_DIM: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
):
HALF = HEAD_DIM // 2
hmask = heads < n_heads
@@ -224,9 +224,9 @@ def _qk_norm_rope_backward(
sin_row_stride,
rope_indices,
seqlen,
- HEAD_DIM: tl.constexpr,
- BLOCK_H: tl.constexpr,
- HAS_ROPE_INDICES: tl.constexpr,
+ HEAD_DIM: tl.constexpr, # hd
+ BLOCK_H: tl.constexpr, # heads per program
+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids
):
row = tl.program_id(0).to(tl.int64)
hblock = tl.program_id(1)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 6e77a3d6112..b52af974ce4 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -27,14 +27,14 @@ def _rms_layernorm_forward(
X,
W,
r,
- Y_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- W_row_stride: tl.constexpr,
- r_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
+ Y_row_stride: tl.constexpr, # Y row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ W_row_stride: tl.constexpr, # W row stride
+ r_row_stride: tl.constexpr, # rstd row stride
+ n_cols: tl.constexpr, # hidden size (or hd)
+ eps: tl.constexpr, # norm epsilon, per model
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
):
"""
Fast RMS Layernorm kernel
@@ -68,22 +68,22 @@ def _rms_layernorm_forward(
@triton.jit
def _rms_backward_row(
dY,
- dY_row_stride: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
dX,
- dX_row_stride: tl.constexpr,
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
X,
- X_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr, # X row stride (hidden)
W,
r,
- r_row_stride: tl.constexpr,
+ r_row_stride: tl.constexpr, # rstd row stride
row_idx,
col_offsets,
mask,
- n_cols: tl.constexpr,
- GEMMA: tl.constexpr,
+ n_cols: tl.constexpr, # hidden size (or hd)
+ GEMMA: tl.constexpr, # gemma (1 + w) weight
dW_acc,
- TRAIN_W: tl.constexpr,
- FROM_Y: tl.constexpr,
+ TRAIN_W: tl.constexpr, # weight grad partials
+ FROM_Y: tl.constexpr, # recompute normed from saved Y
):
# with FROM_Y, X holds Y and normed = Y / W; r still scales dX.
dY += row_idx * dY_row_stride
@@ -129,18 +129,18 @@ def _rms_layernorm_backward(
dW,
n_rows,
rows_per_program,
- dY_row_stride: tl.constexpr,
- dX_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- W_row_stride: tl.constexpr,
- r_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- GEMMA: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- TRAIN_W: tl.constexpr,
- FROM_Y: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ W_row_stride: tl.constexpr, # W row stride
+ r_row_stride: tl.constexpr, # rstd row stride
+ n_cols: tl.constexpr, # hidden size (or hd)
+ eps: tl.constexpr, # norm epsilon, per model
+ GEMMA: tl.constexpr, # gemma (1 + w) weight
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
+ TRAIN_W: tl.constexpr, # weight grad partials
+ FROM_Y: tl.constexpr, # recompute normed from saved Y
):
"""
Fast RMS Layernorm kernel for the backward pass
@@ -219,14 +219,14 @@ def _gemma_rms_layernorm_forward(
X,
W,
r,
- Y_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- W_row_stride: tl.constexpr,
- r_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
+ Y_row_stride: tl.constexpr, # Y row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ W_row_stride: tl.constexpr, # W row stride
+ r_row_stride: tl.constexpr, # rstd row stride
+ n_cols: tl.constexpr, # hidden size (or hd)
+ eps: tl.constexpr, # norm epsilon, per model
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
):
# Copies google-deepmind/gemma layers.py#L31 and keras-nlp gemma/rms_normalization.py#L33 exactly:
# essentially all in float32.
@@ -255,7 +255,9 @@ def _gemma_rms_layernorm_forward(
@triton.jit
-def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):
+def _fold_lanes(
+ x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr
+): # ROWS rows per program, WARPS num_warps of the launch, HALF half of a lane pair
return tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)
@@ -265,7 +267,9 @@ _HIP_ROW_ORDER = tl.constexpr(bool(torch.version.hip))
@triton.jit
-def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+def _row_dot_in_hip_row_order(
+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr
+): # ROWS rows per program, WARPS num_warps of the launch
# Bit-exact replay of the one-row kernels' tl.sum on AMD (WARPS = BLOCK_SIZE // wavefront):
# fma(a_l, b_l, a_(l-8) * b_(l-8)), row_shr 4, 2, 1, 16-lane rows by lane bit 4 then 5, warps.
LANES: tl.constexpr = a.shape[1] // WARPS
@@ -287,7 +291,9 @@ def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
@triton.jit
-def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+def _row_dot_in_cuda_row_order(
+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr
+): # ROWS rows per program, WARPS num_warps of the launch
# Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):
# fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.
a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
@@ -306,7 +312,9 @@ def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
@triton.jit
-def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+def _row_dot_in_row_kernel_order(
+ a, b, ROWS: tl.constexpr, WARPS: tl.constexpr
+): # ROWS rows per program, WARPS num_warps of the launch
if _HIP_ROW_ORDER:
total = _row_dot_in_hip_row_order(a, b, ROWS, WARPS)
else:
@@ -321,15 +329,15 @@ def _rms_layernorm_forward_rows(
W,
r,
n_rows,
- Y_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- GEMMA: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- ROWS: tl.constexpr,
- WARPS: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
+ Y_row_stride: tl.constexpr, # Y row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ n_cols: tl.constexpr, # hidden size (or hd)
+ eps: tl.constexpr, # norm epsilon, per model
+ GEMMA: tl.constexpr, # gemma (1 + w) weight
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
+ ROWS: tl.constexpr, # rows per program
+ WARPS: tl.constexpr, # num_warps of the launch
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
):
pid = tl.program_id(0)
if LONG_INDEXING:
@@ -364,21 +372,21 @@ def _rms_layernorm_forward_rows(
@triton.jit
def _rms_backward_rows_block(
dY,
- dY_row_stride: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
dX,
- dX_row_stride: tl.constexpr,
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
X,
- X_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr, # X row stride (hidden)
W,
r,
rows,
n_rows,
- n_cols: tl.constexpr,
- GEMMA: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- ROWS: tl.constexpr,
- WARPS: tl.constexpr,
- FROM_Y: tl.constexpr,
+ n_cols: tl.constexpr, # hidden size (or hd)
+ GEMMA: tl.constexpr, # gemma (1 + w) weight
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
+ ROWS: tl.constexpr, # rows per program
+ WARPS: tl.constexpr, # num_warps of the launch
+ FROM_Y: tl.constexpr, # recompute normed from saved Y
):
col_offsets = tl.arange(0, BLOCK_SIZE)
row_mask = rows < n_rows
@@ -426,18 +434,18 @@ def _rms_layernorm_backward_rows(
dW,
n_rows,
blocks_per_program,
- dY_row_stride: tl.constexpr,
- dX_row_stride: tl.constexpr,
- X_row_stride: tl.constexpr,
- n_cols: tl.constexpr,
- eps: tl.constexpr,
- GEMMA: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- ROWS: tl.constexpr,
- WARPS: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- TRAIN_W: tl.constexpr,
- FROM_Y: tl.constexpr,
+ dY_row_stride: tl.constexpr, # dY row stride (hidden)
+ dX_row_stride: tl.constexpr, # dX row stride (hidden)
+ X_row_stride: tl.constexpr, # X row stride (hidden)
+ n_cols: tl.constexpr, # hidden size (or hd)
+ eps: tl.constexpr, # norm epsilon, per model
+ GEMMA: tl.constexpr, # gemma (1 + w) weight
+ BLOCK_SIZE: tl.constexpr, # n_cols next pow2
+ ROWS: tl.constexpr, # rows per program
+ WARPS: tl.constexpr, # num_warps of the launch
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ TRAIN_W: tl.constexpr, # weight grad partials
+ FROM_Y: tl.constexpr, # recompute normed from saved Y
):
pid = tl.program_id(0)
if LONG_INDEXING:
@@ -490,7 +498,12 @@ def _rms_layernorm_backward_rows(
@triton.jit(do_not_specialize = ("n_programs",))
def _rms_sum_dw_partials(
- partials, dW, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr
+ partials,
+ dW,
+ n_programs,
+ n_cols,
+ BLOCK_ROWS: tl.constexpr,
+ BLOCK_COLS: tl.constexpr, # BLOCK_ROWS partial rows per step, BLOCK_COLS columns per program
):
# Fixed tile order, so eager and torch.compile give the same bytes.
cols = tl.program_id(0) * BLOCK_COLS + tl.arange(0, BLOCK_COLS)
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index dc5f3962688..57eeb69babe 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -46,12 +46,12 @@ def _rope_heads_tile(
n_heads,
cos1,
sin1,
- half_head_dim: tl.constexpr,
- HEADS: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- CAST_TO_SIN: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BACKWARD_PASS: tl.constexpr,
+ half_head_dim: tl.constexpr, # hd // 2
+ HEADS: tl.constexpr, # heads per program
+ BLOCK_SIZE: tl.constexpr, # hd // 2 next pow2
+ CAST_TO_SIN: tl.constexpr, # compute in sin dtype (fp32 gemma)
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ BACKWARD_PASS: tl.constexpr, # backward (negated sin)
):
# Rotates HEADS heads of one (batch, seq) row from X into OUT (OUT may be X: each element is
# read before it is written by the same program).
@@ -123,15 +123,15 @@ def _rope_embedding_QK(
n_heads_Q: tl.constexpr, # 32
n_heads_K: tl.constexpr, # 8 (0 without K)
n_groups_Q: tl.constexpr, # 1
- half_head_dim: tl.constexpr,
- BACKWARD_PASS: tl.constexpr,
- HAS_ROPE_INDICES: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- CAST_TO_SIN: tl.constexpr,
- HEADS_Q: tl.constexpr,
- HEADS_K: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
- EVICT_INDICES: tl.constexpr,
+ half_head_dim: tl.constexpr, # hd // 2
+ BACKWARD_PASS: tl.constexpr, # backward (negated sin)
+ HAS_ROPE_INDICES: tl.constexpr, # explicit rope position ids
+ LONG_INDEXING: tl.constexpr, # int64 offsets past 2**31
+ CAST_TO_SIN: tl.constexpr, # compute in sin dtype (fp32 gemma)
+ HEADS_Q: tl.constexpr, # query heads per program
+ HEADS_K: tl.constexpr, # key heads per program
+ BLOCK_SIZE: tl.constexpr, # hd // 2 next pow2
+ EVICT_INDICES: tl.constexpr, # evict-first hint on position ids
):
"""
RoPE is Q * cos + rotate_half(Q) * sin; the backward is the same with -sin.
diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py
index 13b2ea33ebc..b82c396c2bf 100644
--- a/unsloth/kernels/swiglu.py
+++ b/unsloth/kernels/swiglu.py
@@ -22,7 +22,9 @@ INT32_SAFETY_BUFFER = NUM_INT32_ELEMENTS - BLOCK_SIZE * SAFE_INT32_BUFFER_MULTIP
@triton.jit
-def _fg_kernel(e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr):
+def _fg_kernel(
+ e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+): # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
block_idx = tl.program_id(0)
if LONG_INDEXING:
offsets = block_idx.to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE).to(tl.int64)
@@ -62,7 +64,9 @@ def swiglu_fg_kernel(e, g):
@triton.jit
-def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr):
+def _DWf_DW_dfg_kernel(
+ DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.constexpr
+): # BLOCK_SIZE elements per program, LONG_INDEXING int64 offsets past 2**31
"""
e = e.float()
se = 1.0 / (1.0 + torch.exp(-e))
diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.py
index c32ab1c942b..3cc107abab8 100644
--- a/unsloth/optimizers/flash_adamw.py
+++ b/unsloth/optimizers/flash_adamw.py
@@ -262,7 +262,9 @@ def _variance_update(var, grad, beta2):
import triton.language as tl
@triton.jit
- def kernel(var_ptr, grad_ptr, out_ptr, n, beta2, BLOCK: tl.constexpr):
+ def kernel(
+ var_ptr, grad_ptr, out_ptr, n, beta2, BLOCK: tl.constexpr
+ ): # elements per program
offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n
v = tl.load(var_ptr + offsets, mask = mask)