Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kernel cache key tag and readback-barrier key for every thread (checkpoint recompute, torch >= 2.12) (#15997)

Daniel Han3cc279b56585
diff --git a/tests/test_compile_config_threads.py b/tests/test_compile_config_threads.py
new file mode 100644
index 00000000000..ef7dcb4d614
--- /dev/null
+++ b/tests/test_compile_config_threads.py
@@ -0,0 +1,193 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""torch >= 2.12 keeps config per thread; a reentrant checkpoint recompiles on autograd's device
+thread, which must see Unsloth's cache_key_tag and readback-barrier pass. CPU only."""
+
+import importlib.util
+import inspect
+import os
+import threading
+import uuid
+from contextvars import ContextVar
+from pathlib import Path
+
+import pytest
+import torch
+import torch._dynamo
+import torch._inductor.config as inductor_config
+import torch.utils.checkpoint as checkpoint_module
+
+pytest.importorskip("triton")
+torch_config = pytest.importorskip("unsloth_zoo.torch_config")
+
+ROOT = Path(__file__).resolve().parents[1]
+_COMPILER_CONFIG = getattr(torch.compiler, "config", None)
+if not hasattr(_COMPILER_CONFIG, "cache_key_tag"):
+ pytest.skip("torch has no compile cache key tag", allow_module_level = True)
+try:
+ from torch._inductor.custom_graph_pass import CustomGraphPass
+except ImportError:
+ pytest.skip("torch has no CustomGraphPass", allow_module_level = True)
+
+_KEYS = [(_COMPILER_CONFIG, "cache_key_tag"), (inductor_config, "post_grad_custom_post_pass")]
+
+
+def _load(relative, name):
+ spec = importlib.util.spec_from_file_location(name, ROOT / relative)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ assert (
+ Path(module.__file__).resolve().is_relative_to(ROOT)
+ ) # this tree, not an installed unsloth
+ return module
+
+
+def _invalidate(module):
+ if hasattr(module, "_is_dirty"):
+ module._is_dirty = True
+ for attr, default in (
+ ("_hash_dirty_var", True),
+ ("_hash_cache_var", None),
+ ("_get_dict_dirty_keys_var", None),
+ ("_get_dict_cache_var", None),
+ ):
+ if isinstance(getattr(module, attr, None), ContextVar):
+ setattr(module, attr, ContextVar(attr, default = default))
+
+
[email protected](autouse = True)
+def _restore_torch_config():
+ saved = []
+ for module, name in _KEYS:
+ entry = module._config[name]
+ override = entry.user_override
+ current = override.get() if isinstance(override, ContextVar) else override
+ saved.append(
+ (module, entry, current, entry.default, entry.env_value_default, entry.env_value_force)
+ )
+ torch._dynamo.reset()
+ yield
+ for module, entry, current, default, env_default, env_force in saved:
+ if isinstance(entry.user_override, ContextVar):
+ entry.user_override.set(current)
+ else:
+ entry.user_override = current
+ entry.default = default
+ entry.env_value_default = env_default
+ entry.env_value_force = env_force
+ _invalidate(module)
+ torch._dynamo.reset()
+
+
+def _in_thread(fn, name = "device"):
+ box = {}
+
+ def run():
+ try:
+ box["value"] = fn()
+ except BaseException as error: # noqa: BLE001
+ box["error"] = error
+
+ thread = threading.Thread(target = run, name = name)
+ thread.start()
+ thread.join()
+ if "error" in box:
+ raise box["error"]
+ return box["value"]
+
+
+def _kernel_tag(tmp_path):
+ triton_launch = _load("unsloth/kernels/triton_launch.py", "unsloth_triton_launch_under_test")
+ kernel = tmp_path / "fake_kernel.py"
+ kernel.write_text(f"# {uuid.uuid4()}\n")
+ triton_launch.tag_compile_cache(str(kernel))
+ tags = _COMPILER_CONFIG.cache_key_tag.split(",")
+ tag = next(t for t in tags if t.startswith(f"unsloth/{os.path.basename(kernel)}:"))
+ return tag
+
+
+def test_kernel_cache_tag_reaches_other_threads(tmp_path):
+ tag = _kernel_tag(tmp_path)
+ assert tag in _in_thread(lambda: _COMPILER_CONFIG.cache_key_tag).split(",")
+
+
+def test_checkpoint_recompute_compiles_with_the_kernel_cache_tag(tmp_path, monkeypatch):
+ original = checkpoint_module.CheckpointFunction.backward
+
+ def device_thread_backward(ctx, *grads):
+ return _in_thread(lambda: original(ctx, *grads))
+
+ monkeypatch.setattr(
+ checkpoint_module.CheckpointFunction, "backward", staticmethod(device_thread_backward)
+ )
+ tag = _kernel_tag(tmp_path)
+ seen = []
+ key = f"unsloth-test-recorder-{uuid.uuid4()}" # a warm FX cache would skip the pass
+
+ class Recorder(CustomGraphPass):
+ def __call__(self, graph):
+ seen.append((threading.current_thread().name, _COMPILER_CONFIG.cache_key_tag))
+
+ def uuid(self):
+ return key
+
+ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", Recorder())
+ compiled = torch.compile(lambda x, w: torch.nn.functional.gelu(x @ w).sum(-1))
+ x = torch.randn(4, 8, requires_grad = True)
+ w = torch.randn(8, 8, requires_grad = True)
+ loss = checkpoint_module.checkpoint(compiled, x, w, use_reentrant = True).sum()
+ loss.backward() # grad-mode guard miss: grad graph + its lazy backward compile on "device"
+ on_device = [tags for name, tags in seen if name == "device"]
+ assert len(on_device) == 2, seen
+ assert all(tag in tags.split(",") for tags in on_device), (tag, on_device)
+
+
+def test_readback_barrier_cache_key_reaches_other_threads():
+ import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")
+ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)
+ import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
+ assert getattr(inductor_config.post_grad_custom_post_pass, "_unsloth_readback_barrier", False)
+ on_device = _in_thread(lambda: inductor_config.post_grad_custom_post_pass)
+ assert getattr(on_device, "_unsloth_readback_barrier", False), on_device
+
+
+def test_later_assignment_reaches_other_threads(monkeypatch):
+ # A pass assigned after Unsloth publishes its own must still reach compile threads.
+ from torch.utils._config_module import ConfigModule
+
+ for attr in ("__setattr__", "patch", "load_config"):
+ if attr in ConfigModule.__dict__:
+ monkeypatch.setattr(ConfigModule, attr, inspect.unwrap(ConfigModule.__dict__[attr]))
+ import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")
+ import_fixes.fix_dynamo_config_thread_visibility()
+ torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)
+ import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
+
+ class AddTwo(CustomGraphPass):
+ def __call__(self, graph):
+ for node in graph.nodes:
+ if node.target is torch.ops.aten.add.Tensor and node.args[1] == 1:
+ node.args = (node.args[0], 2)
+
+ def uuid(self):
+ return f"unsloth-test-add-two-{uuid.uuid4()}"
+
+ later = AddTwo()
+ inductor_config.post_grad_custom_post_pass = later
+ assert _in_thread(lambda: inductor_config.post_grad_custom_post_pass) is later
+ compiled = torch.compile(lambda x: x + 1, fullgraph = True)
+ assert _in_thread(lambda: compiled(torch.tensor([1.0])).item()) == 3.0
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 64088a2dc89..d9b60fe21e2 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -274,9 +274,13 @@ except Exception:
# TORCHINDUCTOR_COMPILE_THREADS.
if os.environ.get("UNSLOTH_FORCE_SINGLE_COMPILE_WORKER", "0") == "1":
try:
- torch._inductor.config.compile_threads = 1
+ from unsloth_zoo.torch_config import set_torch_config
+ set_torch_config(torch._inductor.config, "compile_threads", 1)
except Exception:
- pass
+ try:
+ torch._inductor.config.compile_threads = 1
+ except Exception:
+ pass
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
def _force_single_compile_worker_in_zoo():
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 92c9b07260e..e8149a1bb0b 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -7148,11 +7148,21 @@ def fix_triton_compiled_kernel_missing_attrs():
)
+def _set_torch_config(config_module, name, value):
+ # torch >= 2.12 keeps config assignments per thread; checkpoint recomputes compile on autograd's device thread.
+ try:
+ from unsloth_zoo.torch_config import set_torch_config
+ except Exception:
+ setattr(config_module, name, value)
+ return
+ set_torch_config(config_module, name, value)
+
+
def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass):
# Warm FX / AOTAutograd caches would return barrier-free kernels: a no-op pass's uuid keys them.
if custom_graph_pass is None:
# torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.
- inductor_config.fx_graph_cache = False
+ _set_torch_config(inductor_config, "fx_graph_cache", False)
return
marker = "unsloth-inductor-readback-barrier-v1"
@@ -7167,14 +7177,16 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)
existing = getattr(inductor_config, "post_grad_custom_post_pass", None)
if existing is None:
- inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()
+ _set_torch_config(inductor_config, "post_grad_custom_post_pass", ReadbackBarrierCacheKey())
return
# A pre-set pass keys the cache by its uuid alone; raw callables / uuid-less passes bypass it.
if isinstance(existing, (list, tuple)):
if any(getattr(p, "_unsloth_readback_barrier", False) for p in existing):
return
- inductor_config.post_grad_custom_post_pass = type(existing)(
- [*existing, ReadbackBarrierCacheKey()]
+ _set_torch_config(
+ inductor_config,
+ "post_grad_custom_post_pass",
+ type(existing)([*existing, ReadbackBarrierCacheKey()]),
)
return
if getattr(existing, "_unsloth_readback_barrier", False):
@@ -7203,7 +7215,9 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)
inner_uuid = self.inner.uuid()
return None if inner_uuid is None else (marker, inner_uuid)
- inductor_config.post_grad_custom_post_pass = ReadbackBarrierKeyedPass(existing)
+ _set_torch_config(
+ inductor_config, "post_grad_custom_post_pass", ReadbackBarrierKeyedPass(existing)
+ )
def fix_inductor_reduction_readback_race():
@@ -7289,6 +7303,10 @@ def fix_dynamo_config_thread_visibility():
from contextvars import ContextVar
except Exception:
return
+ try:
+ from torch.utils._config_module import _UNSET_SENTINEL as _unset
+ except Exception:
+ _unset = None
try:
probe = getattr(_dynamo_config, "_config", {}).get("recompile_limit", None)
@@ -7381,6 +7399,9 @@ def fix_dynamo_config_thread_visibility():
entry = self.__dict__["_config"].get(name, None)
if entry is not None and entry.alias is None:
entry.default = value
+ # A process-wide value (set_torch_config, TORCHINDUCTOR_* env) shadows default: replace it too.
+ if _unset is not None and entry.env_value_default is not _unset:
+ entry.env_value_default = value
except Exception:
pass
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 0d7868a06d8..1d564b3d07f 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -10,7 +10,6 @@
# limitations under the License.
import functools
-import hashlib
import os
import warnings
import triton
@@ -1034,16 +1033,7 @@ def _bf16_traceable():
_BF16_TRACEABLE = _TRACEABLE and _bf16_traceable()
-def _tag_compile_cache(path):
- # Inductor's FX cache keys a triton_op without its source; key on the file so upgrades miss.
- config = getattr(torch.compiler, "config", None)
- if config is None or not hasattr(config, "cache_key_tag"):
- return
- with open(path, "rb") as file:
- tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"
- tags = [t for t in config.cache_key_tag.split(",") if t]
- if tag not in tags:
- config.cache_key_tag = ",".join(tags + [tag])
+_tag_compile_cache = triton_launch.tag_compile_cache
def _traced_kernel(kernel):
diff --git a/unsloth/kernels/triton_launch.py b/unsloth/kernels/triton_launch.py
index 68225724162..e1abfe2bfa7 100644
--- a/unsloth/kernels/triton_launch.py
+++ b/unsloth/kernels/triton_launch.py
@@ -115,6 +115,20 @@ def launch(kernel, grid, args, n_tensors, constexprs, device_index, **options):
return None
+try:
+ from unsloth_zoo.torch_config import set_torch_config
+except Exception: # older unsloth_zoo: current thread only
+
+ def set_torch_config(
+ config_module,
+ name,
+ value,
+ force = False,
+ ):
+ setattr(config_module, name, value)
+ return value
+
+
def tag_compile_cache(path):
"""Add a hash of a kernel file to torch.compile's cache key. Inductor's FX graph cache keys a
torch.library.triton_op call without the Triton source behind it, so after an Unsloth upgrade
@@ -126,4 +140,5 @@ def tag_compile_cache(path):
tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"
tags = [t for t in config.cache_key_tag.split(",") if t]
if tag not in tags:
- config.cache_key_tag = ",".join(tags + [tag])
+ # Every thread: a reentrant checkpoint recompiles its grad graph on autograd's device thread.
+ set_torch_config(config, "cache_key_tag", ",".join(tags + [tag]))