Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# Unsloth Zoo - Utilities for Unsloth
# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""torch >= 2.12 keeps config per thread; a reentrant checkpoint recompiles on autograd's device
thread, which must see Unsloth's cache_key_tag and readback-barrier pass. CPU only."""
import importlib.util
import inspect
import os
import threading
import uuid
from contextvars import ContextVar
from pathlib import Path
import pytest
import torch
import torch._dynamo
import torch._inductor.config as inductor_config
import torch.utils.checkpoint as checkpoint_module
pytest.importorskip("triton")
torch_config = pytest.importorskip("unsloth_zoo.torch_config")
ROOT = Path(__file__).resolve().parents[1]
_COMPILER_CONFIG = getattr(torch.compiler, "config", None)
if not hasattr(_COMPILER_CONFIG, "cache_key_tag"):
pytest.skip("torch has no compile cache key tag", allow_module_level = True)
try:
from torch._inductor.custom_graph_pass import CustomGraphPass
except ImportError:
pytest.skip("torch has no CustomGraphPass", allow_module_level = True)
_KEYS = [(_COMPILER_CONFIG, "cache_key_tag"), (inductor_config, "post_grad_custom_post_pass")]
def _load(relative, name):
spec = importlib.util.spec_from_file_location(name, ROOT / relative)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
assert (
Path(module.__file__).resolve().is_relative_to(ROOT)
) # this tree, not an installed unsloth
return module
def _invalidate(module):
if hasattr(module, "_is_dirty"):
module._is_dirty = True
for attr, default in (
("_hash_dirty_var", True),
("_hash_cache_var", None),
("_get_dict_dirty_keys_var", None),
("_get_dict_cache_var", None),
):
if isinstance(getattr(module, attr, None), ContextVar):
setattr(module, attr, ContextVar(attr, default = default))
@pytest.fixture(autouse = True)
def _restore_torch_config():
saved = []
for module, name in _KEYS:
entry = module._config[name]
override = entry.user_override
current = override.get() if isinstance(override, ContextVar) else override
saved.append(
(module, entry, current, entry.default, entry.env_value_default, entry.env_value_force)
)
torch._dynamo.reset()
yield
for module, entry, current, default, env_default, env_force in saved:
if isinstance(entry.user_override, ContextVar):
entry.user_override.set(current)
else:
entry.user_override = current
entry.default = default
entry.env_value_default = env_default
entry.env_value_force = env_force
_invalidate(module)
torch._dynamo.reset()
def _in_thread(fn, name = "device"):
box = {}
def run():
try:
box["value"] = fn()
except BaseException as error: # noqa: BLE001
box["error"] = error
thread = threading.Thread(target = run, name = name)
thread.start()
thread.join()
if "error" in box:
raise box["error"]
return box["value"]
def _kernel_tag(tmp_path):
triton_launch = _load("unsloth/kernels/triton_launch.py", "unsloth_triton_launch_under_test")
kernel = tmp_path / "fake_kernel.py"
kernel.write_text(f"# {uuid.uuid4()}\n")
triton_launch.tag_compile_cache(str(kernel))
tags = _COMPILER_CONFIG.cache_key_tag.split(",")
tag = next(t for t in tags if t.startswith(f"unsloth/{os.path.basename(kernel)}:"))
return tag
def test_kernel_cache_tag_reaches_other_threads(tmp_path):
tag = _kernel_tag(tmp_path)
assert tag in _in_thread(lambda: _COMPILER_CONFIG.cache_key_tag).split(",")
def test_checkpoint_recompute_compiles_with_the_kernel_cache_tag(tmp_path, monkeypatch):
original = checkpoint_module.CheckpointFunction.backward
def device_thread_backward(ctx, *grads):
return _in_thread(lambda: original(ctx, *grads))
monkeypatch.setattr(
checkpoint_module.CheckpointFunction, "backward", staticmethod(device_thread_backward)
)
tag = _kernel_tag(tmp_path)
seen = []
key = f"unsloth-test-recorder-{uuid.uuid4()}" # a warm FX cache would skip the pass
class Recorder(CustomGraphPass):
def __call__(self, graph):
seen.append((threading.current_thread().name, _COMPILER_CONFIG.cache_key_tag))
def uuid(self):
return key
torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", Recorder())
compiled = torch.compile(lambda x, w: torch.nn.functional.gelu(x @ w).sum(-1))
x = torch.randn(4, 8, requires_grad = True)
w = torch.randn(8, 8, requires_grad = True)
loss = checkpoint_module.checkpoint(compiled, x, w, use_reentrant = True).sum()
loss.backward() # grad-mode guard miss: grad graph + its lazy backward compile on "device"
on_device = [tags for name, tags in seen if name == "device"]
assert len(on_device) == 2, seen
assert all(tag in tags.split(",") for tags in on_device), (tag, on_device)
def test_readback_barrier_cache_key_reaches_other_threads():
import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")
torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)
import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
assert getattr(inductor_config.post_grad_custom_post_pass, "_unsloth_readback_barrier", False)
on_device = _in_thread(lambda: inductor_config.post_grad_custom_post_pass)
assert getattr(on_device, "_unsloth_readback_barrier", False), on_device
def test_later_assignment_reaches_other_threads(monkeypatch):
# A pass assigned after Unsloth publishes its own must still reach compile threads.
from torch.utils._config_module import ConfigModule
for attr in ("__setattr__", "patch", "load_config"):
if attr in ConfigModule.__dict__:
monkeypatch.setattr(ConfigModule, attr, inspect.unwrap(ConfigModule.__dict__[attr]))
import_fixes = _load("unsloth/import_fixes.py", "unsloth_import_fixes_under_test")
import_fixes.fix_dynamo_config_thread_visibility()
torch_config.set_torch_config(inductor_config, "post_grad_custom_post_pass", None)
import_fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
class AddTwo(CustomGraphPass):
def __call__(self, graph):
for node in graph.nodes:
if node.target is torch.ops.aten.add.Tensor and node.args[1] == 1:
node.args = (node.args[0], 2)
def uuid(self):
return f"unsloth-test-add-two-{uuid.uuid4()}"
later = AddTwo()
inductor_config.post_grad_custom_post_pass = later
assert _in_thread(lambda: inductor_config.post_grad_custom_post_pass) is later
compiled = torch.compile(lambda x: x + 1, fullgraph = True)
assert _in_thread(lambda: compiled(torch.tensor([1.0])).item()) == 3.0