Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stream the chunked cross entropy forward logits with evict_first (#13893)
Daniel Han945e2fcf8303
diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.pynew file mode 100644index 00000000000..aaa9be81c64--- /dev/null+++ b/tests/test_cross_entropy_eviction_hints.py@@ -0,0 +1,73 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""The chunked (vocab > 65536) cross entropy forward streams its logits chunk with evict_first from+Volta on, and drops the hint below sm_70 where ptxas rejects it. Compile-only with Triton's own+ptxas: no launch."""++import pytest+import torch++if not torch.cuda.is_available() or torch.version.hip is not None:+ pytest.skip("needs an NVIDIA CUDA device", allow_module_level = True)++triton = pytest.importorskip("triton")+from triton.backends.compiler import GPUTarget+from triton.compiler import ASTSource++from unsloth.kernels import cross_entropy_loss as ce_mod+from unsloth.kernels import rope_embedding as rope_mod++_SIGNATURE = {+ "logits_ptr": "*bf16",+ "loss_ptr": "*fp32",+ "logsumexp_ptr": "*fp32",+ "labels_ptr": "*i64",+}++_COMMON = dict(DO_SOFTCAPPING = False, SOFTCAP = 0.0, DO_LOGIT_SCALING = False, LOGIT_SCALE = 0.0)++_CHUNKED = dict(logits_row_stride = 128256, VOCAB_SIZE = 128256, N_CHUNKS = 2, BLOCK_SIZE = 65536)+++def _policy(capability, monkeypatch):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)+ rope_mod._eviction_hints_ok_at.cache_clear()+ try:+ ce_mod._chunked_evict_at.cache_clear()+ return ce_mod._chunked_evict_policy(torch.device("cuda", 0))+ finally:+ rope_mod._eviction_hints_ok_at.cache_clear()+ ce_mod._chunked_evict_at.cache_clear()+++def _compile(evict, cc):+ fn = ce_mod._chunked_cross_entropy_forward.fn+ signature = {name: _SIGNATURE.get(name, "constexpr") for name in fn.arg_names}+ constexprs = dict(_CHUNKED, EVICT = evict, **_COMMON)+ return triton.compile(+ ASTSource(fn, signature = signature, constexprs = constexprs),+ target = GPUTarget("cuda", cc, 32),+ options = {"num_warps": 32},+ )++[email protected]("capability", [(5, 2), (6, 1)])+def test_chunked_cross_entropy_assembles_before_volta(capability, monkeypatch):+ evict = _policy(capability, monkeypatch)+ assert evict == ""+ cc = capability[0] * 10 + capability[1]+ assert _compile(evict, cc).asm["cubin"]++[email protected]("capability", [(10, 0), (10, 3)])+def test_chunked_cross_entropy_skips_the_hint_on_sm100(capability, monkeypatch):+ # Measured 1.25% slower on B200 with the hint.+ assert _policy(capability, monkeypatch) == ""++[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (12, 0)])+def test_chunked_cross_entropy_streams_logits_from_volta(capability, monkeypatch):+ evict = _policy(capability, monkeypatch)+ assert evict == "evict_first"+ assert "ld.global.L1::evict_first" in _compile(evict, 75).asm["ptx"]diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 38fc705dddc..5d1cf3f1495 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -9,6 +9,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import functoolsimport tritonimport triton.language as tlimport torch@@ -20,6 +21,7 @@ from .utils import (torch_gpu_device,is_cdna,)+from .rope_embedding import _eviction_hints_okfrom transformers.models.llama.modeling_llama import loggerfrom unsloth_zoo.utils import Version@@ -111,6 +113,21 @@ _cross_entropy_forward = triton.heuristics()(_cross_entropy_forward)[email protected]_cache(maxsize = None)+def _chunked_evict_at(device_type, index):+ # Chunks are read once: evict_first is faster except on B200 (sm_100); ROCm unmeasured.+ if device_type != "cuda" or torch.version.hip is not None:+ return ""+ return "" if torch.cuda.get_device_capability(index)[0] == 10 else "evict_first"+++def _chunked_evict_policy(device):+ if not _eviction_hints_ok(device):+ return ""+ index = device.index if device.index is not None else torch.cuda.current_device()+ return _chunked_evict_at(device.type, index)++def _chunked_cross_entropy_forward(logits_ptr,logits_row_stride: tl.constexpr,@@ -124,6 +141,7 @@ def _chunked_cross_entropy_forward(SOFTCAP: tl.constexpr,DO_LOGIT_SCALING: tl.constexpr,LOGIT_SCALE: tl.constexpr,+ EVICT: tl.constexpr,):"""256K vocab divided in 4 chunks@@ -160,7 +178,9 @@ def _chunked_cross_entropy_forward(mask = col_offsets < VOCAB_SIZElabel_idx = tl.load(labels_ptr).to(tl.int32)- logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)+ logits = tl.load(+ logits_ptr + col_offsets, mask = mask, other = -float("inf"), eviction_policy = EVICT+ ).to(tl.float32)# Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).# Do logit scaling for Cohere@@ -365,6 +385,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):SOFTCAP = logit_softcapping,DO_LOGIT_SCALING = DO_LOGIT_SCALING,LOGIT_SCALE = logit_scaling,+ EVICT = _chunked_evict_policy(device),num_warps = 32 if not is_cdna() else 16,enable_fp_fusion = True,)