Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stream the chunked cross entropy forward logits with evict_first (#13893)

Daniel Han945e2fcf8303
diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.py
new file mode 100644
index 00000000000..aaa9be81c64
--- /dev/null
+++ b/tests/test_cross_entropy_eviction_hints.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""The chunked (vocab > 65536) cross entropy forward streams its logits chunk with evict_first from
+Volta on, and drops the hint below sm_70 where ptxas rejects it. Compile-only with Triton's own
+ptxas: no launch."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available() or torch.version.hip is not None:
+ pytest.skip("needs an NVIDIA CUDA device", allow_module_level = True)
+
+triton = pytest.importorskip("triton")
+from triton.backends.compiler import GPUTarget
+from triton.compiler import ASTSource
+
+from unsloth.kernels import cross_entropy_loss as ce_mod
+from unsloth.kernels import rope_embedding as rope_mod
+
+_SIGNATURE = {
+ "logits_ptr": "*bf16",
+ "loss_ptr": "*fp32",
+ "logsumexp_ptr": "*fp32",
+ "labels_ptr": "*i64",
+}
+
+_COMMON = dict(DO_SOFTCAPPING = False, SOFTCAP = 0.0, DO_LOGIT_SCALING = False, LOGIT_SCALE = 0.0)
+
+_CHUNKED = dict(logits_row_stride = 128256, VOCAB_SIZE = 128256, N_CHUNKS = 2, BLOCK_SIZE = 65536)
+
+
+def _policy(capability, monkeypatch):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)
+ rope_mod._eviction_hints_ok_at.cache_clear()
+ try:
+ ce_mod._chunked_evict_at.cache_clear()
+ return ce_mod._chunked_evict_policy(torch.device("cuda", 0))
+ finally:
+ rope_mod._eviction_hints_ok_at.cache_clear()
+ ce_mod._chunked_evict_at.cache_clear()
+
+
+def _compile(evict, cc):
+ fn = ce_mod._chunked_cross_entropy_forward.fn
+ signature = {name: _SIGNATURE.get(name, "constexpr") for name in fn.arg_names}
+ constexprs = dict(_CHUNKED, EVICT = evict, **_COMMON)
+ return triton.compile(
+ ASTSource(fn, signature = signature, constexprs = constexprs),
+ target = GPUTarget("cuda", cc, 32),
+ options = {"num_warps": 32},
+ )
+
+
[email protected]("capability", [(5, 2), (6, 1)])
+def test_chunked_cross_entropy_assembles_before_volta(capability, monkeypatch):
+ evict = _policy(capability, monkeypatch)
+ assert evict == ""
+ cc = capability[0] * 10 + capability[1]
+ assert _compile(evict, cc).asm["cubin"]
+
+
[email protected]("capability", [(10, 0), (10, 3)])
+def test_chunked_cross_entropy_skips_the_hint_on_sm100(capability, monkeypatch):
+ # Measured 1.25% slower on B200 with the hint.
+ assert _policy(capability, monkeypatch) == ""
+
+
[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (12, 0)])
+def test_chunked_cross_entropy_streams_logits_from_volta(capability, monkeypatch):
+ evict = _policy(capability, monkeypatch)
+ assert evict == "evict_first"
+ assert "ld.global.L1::evict_first" in _compile(evict, 75).asm["ptx"]
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 38fc705dddc..5d1cf3f1495 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -9,6 +9,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import functools
import triton
import triton.language as tl
import torch
@@ -20,6 +21,7 @@ from .utils import (
torch_gpu_device,
is_cdna,
)
+from .rope_embedding import _eviction_hints_ok
from transformers.models.llama.modeling_llama import logger
from unsloth_zoo.utils import Version
@@ -111,6 +113,21 @@ _cross_entropy_forward = triton.heuristics(
)(_cross_entropy_forward)
[email protected]_cache(maxsize = None)
+def _chunked_evict_at(device_type, index):
+ # Chunks are read once: evict_first is faster except on B200 (sm_100); ROCm unmeasured.
+ if device_type != "cuda" or torch.version.hip is not None:
+ return ""
+ return "" if torch.cuda.get_device_capability(index)[0] == 10 else "evict_first"
+
+
+def _chunked_evict_policy(device):
+ if not _eviction_hints_ok(device):
+ return ""
+ index = device.index if device.index is not None else torch.cuda.current_device()
+ return _chunked_evict_at(device.type, index)
+
+
def _chunked_cross_entropy_forward(
logits_ptr,
logits_row_stride: tl.constexpr,
@@ -124,6 +141,7 @@ def _chunked_cross_entropy_forward(
SOFTCAP: tl.constexpr,
DO_LOGIT_SCALING: tl.constexpr,
LOGIT_SCALE: tl.constexpr,
+ EVICT: tl.constexpr,
):
"""
256K vocab divided in 4 chunks
@@ -160,7 +178,9 @@ def _chunked_cross_entropy_forward(
mask = col_offsets < VOCAB_SIZE
label_idx = tl.load(labels_ptr).to(tl.int32)
- logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)
+ logits = tl.load(
+ logits_ptr + col_offsets, mask = mask, other = -float("inf"), eviction_policy = EVICT
+ ).to(tl.float32)
# Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).
# Do logit scaling for Cohere
@@ -365,6 +385,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
SOFTCAP = logit_softcapping,
DO_LOGIT_SCALING = DO_LOGIT_SCALING,
LOGIT_SCALE = logit_scaling,
+ EVICT = _chunked_evict_policy(device),
num_warps = 32 if not is_cdna() else 16,
enable_fp_fusion = True,
)