Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kernel cache key tag and readback-barrier key for every thread (checkpoint recompute, torch >= 2.12) (#15997)
Daniel Han3cc279b56585
diff --git a/tests/test_compile_config_threads.py b/tests/test_compile_config_threads.pynew file mode 100644index 00000000000..ef7dcb4d614--- /dev/null+++ b/tests/test_compile_config_threads.py@@ -0,0 +1,193 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""torch >= 2.12 keeps config per thread; a reentrant checkpoint recompiles on autograd's device+thread, which must see Unsloth's cache_key_tag and readback-barrier pass. CPU only."""++import importlib.util+import inspect+import os+import threading+import uuid+from contextvars import ContextVar+from pathlib import Path++import pytest+import torch+import torch._dynamo+import torch._inductor.config as inductor_config+import torch.utils.checkpoint as checkpoint_module++pytest.importorskip("triton")+torch_config = pytest.importorskip("unsloth_zoo.torch_config")++ROOT = Path(__file__).resolve().parents[1]+_COMPILER_CONFIG = getattr(torch.compiler, "config", None)+if not hasattr(_COMPILER_CONFIG, "cache_key_tag"):+ pytest.skip("torch has no compile cache key tag", allow_module_level = True)+try:+ from torch._inductor.custom_graph_pass import CustomGraphPass+except ImportError:+ pytest.skip("torch has no CustomGraphPass", allow_module_level = True)++_KEYS = [(_COMPILER_CONFIG, "cache_key_tag"), (inductor_config, "post_grad_custom_post_pass")]+++def _load(relative, name):+ spec = importlib.util.spec_from_file_location(name, ROOT / relative)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ assert (+ Path(module.__file__).resolve().is_relative_to(ROOT)+ ) # this tree, not an installed unsloth+ return module+++def _invalidate(module):+ if hasattr(module, "_is_dirty"):+ module._is_dirty = True+ for attr, default in (+ ("_hash_dirty_var", True),+ ("_hash_cache_var", None),+ ("_get_dict_dirty_keys_var", None),+ ("_get_dict_cache_var", None),+ ):+ if isinstance(getattr(module, attr, None), ContextVar):+ setattr(module, attr, ContextVar(attr, default = default))++[email protected](autouse = True)+def _restore_torch_config():+ saved = []+ for module, name in _KEYS:+ entry = module._config[name]+ override = entry.user_override+ current = override.get() if isinstance(override, ContextVar) else override+ saved.append(+ (module, entry, current, entry.default, entry.env_value_default, entry.env_value_force)+ )+ torch._dynamo.reset()+ yield+ for module, entry, current, default, env_default, env_force in saved:+ if isinstance(entry.user_override, ContextVar):+ entry.user_override.set(current)+ else:+ entry.user_override = current+ entry.default = default+ entry.env_value_default = env_default+ entry.env_value_force = env_force+ _invalidate(module)+ torch._dynamo.reset()+++def _in_thread(fn, name = "device"):+ box = {}++ def run():+ try:+ box["value"] = fn()+ except BaseException as error: # noqa: BLE001+ box["error"] = error++ thread = threading.Thread(target = run, name = name)+ thread.start()+ thread.join()+ if "error" in box:+ raise box["error"]+ return box["value"]+++def _kernel_tag(tmp_path):+ triton_launch = _load("unsloth/kernels/triton_launch.py", "unsloth_triton_launch_under_test")+ kernel = tmp_path / "fake_kernel.py"+ kernel.write_text(f"# {uuid.uuid4()}\n")+ triton_launch.tag_compile_cache(str(kernel))+ tags = _COMPILER_CONFIG.cache_key_tag.split(",")+ tag = next(t for t in tags if t.startswith(f"unsloth/{os.path.basename(kernel)}:"))+ return tag+++def test_kernel_cache_tag_reaches_other_threads(tmp_path):+ tag = _kernel_tag(tmp_path)+ assert tag in _in_thread(lambda: _COMPILER_CONFIG.cache_key_tag).split(",")+++def test_checkpoint_recompute_compiles_with_the_kernel_cache_tag(tmp_path, monkeypatch):+ original = checkpoint_module.CheckpointFunction.backward++ def device_thread_backward(ctx, *grads):+ return _in_thread(lambda: original(ctx, *grads))++ monkeypatch.setattr(+ checkpoint_module.CheckpointFunction, "backward", staticmethod(device_thread_backward)+ )+ tag = _kernel_tag(tmp_path)+ seen = []+ key = f"unsloth-test-recorder-{uuid.uuid4()}" # a warm FX cache would skip the pass++ class Recorder(CustomGraphPass):+ def __call__(self, graph):+ seen.append((threading.current_thread().name, _COMPILER_CONFIG.cache_key_tag))++ def uuid(self):+ return key++ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", Recorder())+ compiled = torch.compile(lambda x, w: torch.nn.functional.gelu(x @ w).sum(-1))+ x = torch.randn(4, 8, requires_grad = True)+ w = torch.randn(8, 8, requires_grad = True)+ loss = checkpoint_module.checkpoint(compiled, x, w, use_reentrant = True).sum()+ loss.backward() # grad-mode guard miss: grad graph + its lazy backward compile on "device"+ on_device = [tags for name, tags in seen if name == "device"]+ assert len(on_device) == 2, seen+ assert all(tag in tags.split(",") for tags in on_device), (tag, on_device)+++def test_readback_barrier_cache_key_reaches_other_threads():+ import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")+ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)+ import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)+ assert getattr(inductor_config.post_grad_custom_post_pass, "_unsloth_readback_barrier", False)+ on_device = _in_thread(lambda: inductor_config.post_grad_custom_post_pass)+ assert getattr(on_device, "_unsloth_readback_barrier", False), on_device+++def test_later_assignment_reaches_other_threads(monkeypatch):+ # A pass assigned after Unsloth publishes its own must still reach compile threads.+ from torch.utils._config_module import ConfigModule++ for attr in ("__setattr__", "patch", "load_config"):+ if attr in ConfigModule.__dict__:+ monkeypatch.setattr(ConfigModule, attr, inspect.unwrap(ConfigModule.__dict__[attr]))+ import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")+ import_fixes.fix_dynamo_config_thread_visibility()+ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)+ import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)++ class AddTwo(CustomGraphPass):+ def __call__(self, graph):+ for node in graph.nodes:+ if node.target is torch.ops.aten.add.Tensor and node.args[1] == 1:+ node.args = (node.args[0], 2)++ def uuid(self):+ return f"unsloth-test-add-two-{uuid.uuid4()}"++ later = AddTwo()+ inductor_config.post_grad_custom_post_pass = later+ assert _in_thread(lambda: inductor_config.post_grad_custom_post_pass) is later+ compiled = torch.compile(lambda x: x + 1, fullgraph = True)+ assert _in_thread(lambda: compiled(torch.tensor([1.0])).item()) == 3.0diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 64088a2dc89..d9b60fe21e2 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -274,9 +274,13 @@ except Exception:# TORCHINDUCTOR_COMPILE_THREADS.if os.environ.get("UNSLOTH_FORCE_SINGLE_COMPILE_WORKER", "0") == "1":try:- torch._inductor.config.compile_threads = 1+ from unsloth_zoo.torch_config import set_torch_config+ set_torch_config(torch._inductor.config, "compile_threads", 1)except Exception:- pass+ try:+ torch._inductor.config.compile_threads = 1+ except Exception:+ passos.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"def _force_single_compile_worker_in_zoo():diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 92c9b07260e..e8149a1bb0b 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -7148,11 +7148,21 @@ def fix_triton_compiled_kernel_missing_attrs():)+def _set_torch_config(config_module, name, value):+ # torch >= 2.12 keeps config assignments per thread; checkpoint recomputes compile on autograd's device thread.+ try:+ from unsloth_zoo.torch_config import set_torch_config+ except Exception:+ setattr(config_module, name, value)+ return+ set_torch_config(config_module, name, value)++def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass):# Warm FX / AOTAutograd caches would return barrier-free kernels: a no-op pass's uuid keys them.if custom_graph_pass is None:# torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.- inductor_config.fx_graph_cache = False+ _set_torch_config(inductor_config, "fx_graph_cache", False)returnmarker = "unsloth-inductor-readback-barrier-v1"@@ -7167,14 +7177,16 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)existing = getattr(inductor_config, "post_grad_custom_post_pass", None)if existing is None:- inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()+ _set_torch_config(inductor_config, "post_grad_custom_post_pass", ReadbackBarrierCacheKey())return# A pre-set pass keys the cache by its uuid alone; raw callables / uuid-less passes bypass it.if isinstance(existing, (list, tuple)):if any(getattr(p, "_unsloth_readback_barrier", False) for p in existing):return- inductor_config.post_grad_custom_post_pass = type(existing)(- [*existing, ReadbackBarrierCacheKey()]+ _set_torch_config(+ inductor_config,+ "post_grad_custom_post_pass",+ type(existing)([*existing, ReadbackBarrierCacheKey()]),)returnif getattr(existing, "_unsloth_readback_barrier", False):@@ -7203,7 +7215,9 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)inner_uuid = self.inner.uuid()return None if inner_uuid is None else (marker, inner_uuid)- inductor_config.post_grad_custom_post_pass = ReadbackBarrierKeyedPass(existing)+ _set_torch_config(+ inductor_config, "post_grad_custom_post_pass", ReadbackBarrierKeyedPass(existing)+ )def fix_inductor_reduction_readback_race():@@ -7289,6 +7303,10 @@ def fix_dynamo_config_thread_visibility():from contextvars import ContextVarexcept Exception:return+ try:+ from torch.utils._config_module import _UNSET_SENTINEL as _unset+ except Exception:+ _unset = Nonetry:probe = getattr(_dynamo_config, "_config", {}).get("recompile_limit", None)@@ -7381,6 +7399,9 @@ def fix_dynamo_config_thread_visibility():entry = self.__dict__["_config"].get(name, None)if entry is not None and entry.alias is None:entry.default = value+ # A process-wide value (set_torch_config, TORCHINDUCTOR_* env) shadows default: replace it too.+ if _unset is not None and entry.env_value_default is not _unset:+ entry.env_value_default = valueexcept Exception:passdiff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 0d7868a06d8..1d564b3d07f 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -10,7 +10,6 @@# limitations under the License.import functools-import hashlibimport osimport warningsimport triton@@ -1034,16 +1033,7 @@ def _bf16_traceable():_BF16_TRACEABLE = _TRACEABLE and _bf16_traceable()-def _tag_compile_cache(path):- # Inductor's FX cache keys a triton_op without its source; key on the file so upgrades miss.- config = getattr(torch.compiler, "config", None)- if config is None or not hasattr(config, "cache_key_tag"):- return- with open(path, "rb") as file:- tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"- tags = [t for t in config.cache_key_tag.split(",") if t]- if tag not in tags:- config.cache_key_tag = ",".join(tags + [tag])+_tag_compile_cache = triton_launch.tag_compile_cachedef _traced_kernel(kernel):diff --git a/unsloth/kernels/triton_launch.py b/unsloth/kernels/triton_launch.pyindex 68225724162..e1abfe2bfa7 100644--- a/unsloth/kernels/triton_launch.py+++ b/unsloth/kernels/triton_launch.py@@ -115,6 +115,20 @@ def launch(kernel, grid, args, n_tensors, constexprs, device_index, **options):return None+try:+ from unsloth_zoo.torch_config import set_torch_config+except Exception: # older unsloth_zoo: current thread only++ def set_torch_config(+ config_module,+ name,+ value,+ force = False,+ ):+ setattr(config_module, name, value)+ return value++def tag_compile_cache(path):"""Add a hash of a kernel file to torch.compile's cache key. Inductor's FX graph cache keys atorch.library.triton_op call without the Triton source behind it, so after an Unsloth upgrade@@ -126,4 +140,5 @@ def tag_compile_cache(path):tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"tags = [t for t in config.cache_key_tag.split(",") if t]if tag not in tags:- config.cache_key_tag = ",".join(tags + [tag])+ # Every thread: a reentrant checkpoint recompiles its grad graph on autograd's device thread.+ set_torch_config(config, "cache_key_tag", ",".join(tags + [tag]))