Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: each model's compile bundle holds only its own artifacts (#17110)
Daniel Hand4f72beff577
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex baacc49d799..af60fb6b96f 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -29,6 +29,8 @@ _ENV_DIR = "UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR"_ENV_SAVE = "UNSLOTH_DIFFUSION_COMPILE_CACHE_SAVE"_ENV_SYNC = "UNSLOTH_DIFFUSION_COMPILE_CACHE_SYNC"_ENV_MAX_GB = "UNSLOTH_DIFFUSION_COMPILE_CACHE_MAX_GB"+# "0": keep torch's process-wide artifact collector, so each bundle also carries earlier models' artifacts.+_ENV_ISOLATE = "UNSLOTH_DIFFUSION_COMPILE_CACHE_ISOLATE"_DEFAULT_MAX_GB = 20.0_LEGACY_ROOT = Path.home() / ".cache" / "unsloth" / "diffusion_compile_cache"@@ -509,6 +511,7 @@ class CacheContext:# Recompiled artifacts often hash the same, so overwrite instead of the exists() shortcut.rejected_bundle: Optional[str] = Nonelast_touch: float = 0.0+ forwards: frozenset = frozenset()def begin(@@ -560,14 +563,18 @@ def begin(env_fp = env_fp,model_fp = model_fp,mode = mode,+ forwards = _module_forwards(transformer),)# Same reason as restore(): the previous load's save reads the inductor dir that is about to be repointed.- if not wait_for_saves():+ drained = wait_for_saves()+ if not drained:_warn(logger,f"compile-cache: background save still running after {_SAVE_JOIN_TIMEOUT:.0f}s; continuing",)+ else:+ _fresh_collector(logger, ctx.forwards)try:cdir.mkdir(parents = True, exist_ok = True)@@ -835,6 +842,202 @@ def _compile_lock() -> Any:return contextlib.nullcontext()+def _isolation_off() -> bool:+ return (os.environ.get(_ENV_ISOLATE) or "").strip().lower() in ("0", "off", "false", "no")+++def _module_forwards(transformer: Any) -> frozenset:+ """``forward`` codes of ``transformer``'s module classes; Dynamo shares compiled code per code object."""+ codes = set()+ try:+ modules = transformer.modules() if callable(getattr(transformer, "modules", None)) else ()+ for module in modules:+ code = getattr(getattr(type(module), "forward", None), "__code__", None)+ if code is not None:+ codes.add(code)+ except Exception: # noqa: BLE001+ pass+ return frozenset(codes)+++def _dynamo_entries() -> Optional[dict[int, tuple[Any, tuple]]]:+ """``id(code) -> (code, each cache entry's compiled code)``; None when torch does not expose them."""+ try:+ from torch._dynamo import convert_frame # noqa: PLC0415+ from torch._dynamo.eval_frame import _debug_get_cache_entry_list # noqa: PLC0415++ entries = {}+ for ref in list(convert_frame.input_codes.seen):+ code = ref()+ if code is None:+ continue+ compiled = tuple(entry.code for entry in _debug_get_cache_entry_list(code))+ if compiled:+ entries[id(code)] = (code, compiled)+ return entries+ except Exception: # noqa: BLE001+ return None+++_RESUME_RE = re.compile(r"torch_dynamo_resume_in_(.+)_at_\d+")+++def _serves(code: Any, forwards: Any) -> bool:+ """``code`` is one of ``forwards`` or a graph-break resume function of one (matched by name and file)."""+ if code in forwards:+ return True+ name = code.co_name+ resumed = False+ while (match := _RESUME_RE.fullmatch(name)) is not None:+ name, resumed = match.group(1), True+ return resumed and any(+ f.co_name == name and f.co_filename == code.co_filename for f in forwards+ )+++def _still_compiled(collection: "_Collection") -> bool:+ """Dynamo still holds an entry compiled during ``collection``."""+ try:+ from torch._dynamo.eval_frame import _debug_get_cache_entry_list # noqa: PLC0415+ ids = {id(c) for c in collection.compiled}+ return any(+ id(entry.code) in ids+ for code in collection.codes+ for entry in _debug_get_cache_entry_list(code)+ )+ except Exception: # noqa: BLE001 - cannot tell, so keep it+ return True+++class _Collection:+ """Artifacts torch collected between two resets, kept while Dynamo can still run what was compiled then."""++ data: bytes+ codes: frozenset+ compiled: tuple+ # some compiled code is no forward of the models served, so any model may run it+ any_model: bool+++_collections_lock = threading.Lock()+_collections: list[_Collection] = []+# the open collection: Dynamo's entries at its start, and the forwards it serves+_collecting_since: Optional[dict[int, tuple[Any, tuple]]] = None+_collecting_forwards: set = set()+# past this the collector just keeps accumulating, as torch does+_COLLECTIONS_MAX_BYTES = 2 << 30+++def _fresh_collector(logger: Any, forwards: frozenset = frozenset()) -> bool:+ """Reset torch's artifact collector so this model's bundle holds its own artifacts. A Dynamo in-memory hit+ records nothing, so the previous collection is kept for ``save`` to add back; skipped while another bundle is live."""+ global _collecting_since+ with _collections_lock:+ _collecting_forwards.update(forwards)+ if _isolation_off():+ return False+ with _live_lock:+ if _live_dirs:+ return False+ try:+ import torch # noqa: PLC0415+ from torch.compiler._cache import CacheArtifactManager # noqa: PLC0415+ from torch.utils._appending_byte_serializer import AppendingByteSerializer # noqa: F401, PLC0415+ except ImportError:+ # torch 2.7 bundles are a flat list that cannot be merged+ return False+ lock = _compile_lock()+ acquire = getattr(lock, "acquire", None)+ # never wait: a running compile's artifacts are not this model's to drop+ if acquire is not None and not acquire(blocking = False):+ return False+ try:+ entries = _dynamo_entries()+ if entries is None:+ return False+ with _collections_lock:+ since = _collecting_since if _collecting_since is not None else {}+ # by identity: a reset then recompile keeps the count; ``since`` keeps the ids alive+ new = {}+ for key, (code, compiled) in entries.items():+ before = {id(c) for c in since.get(key, (None, ()))[1]}+ fresh = tuple(c for c in compiled if id(c) not in before)+ if fresh:+ new[code] = fresh+ grown = frozenset(new)+ _collections[:] = [c for c in _collections if _still_compiled(c)]+ if grown:+ result = torch.compiler.save_cache_artifacts()+ data = result[0] if result else None+ if data:+ if sum(len(c.data) for c in _collections) + len(data) > _COLLECTIONS_MAX_BYTES:+ return False+ _collections.append(+ _Collection(+ data = data,+ codes = grown,+ compiled = tuple(c for fresh in new.values() for c in fresh),+ any_model = not all(+ _serves(code, _collecting_forwards) for code in grown+ ),+ )+ )+ # no new entry: nothing from this collection runs without recompiling, which records again+ CacheArtifactManager.clear()+ _collecting_since = entries+ _collecting_forwards.clear()+ _collecting_forwards.update(forwards)+ return True+ except Exception as exc: # noqa: BLE001 - keep torch's collector as it is+ _warn(logger, f"compile-cache: could not reset the artifact collector: {exc}")+ return False+ finally:+ if acquire is not None:+ lock.release()+++def _with_reused_collections(+ ctx: CacheContext, data: Optional[bytes], logger: Any+) -> Optional[bytes]:+ """``data`` plus each kept collection ``ctx``'s model may run, merged; None when there is nothing."""+ with _collections_lock:+ reused = [+ c.data+ for c in _collections+ if (c.any_model or any(_serves(code, ctx.forwards) for code in c.codes))+ and _still_compiled(c)+ ]+ if not reused:+ return None if data is None else _one_group_per_type(data, logger)+ return _merge_groups(reused + ([data] if data else []), logger)+++def _merge_groups(bundles: list[bytes], logger: Any) -> Optional[bytes]:+ """Serialized collections, oldest first, as one bundle."""+ try:+ from torch.compiler._cache import ( # noqa: PLC0415+ CacheArtifactManager,+ _deserialize_single_cache,+ _serialize_single_cache,+ )+ from torch.utils._appending_byte_serializer import AppendingByteSerializer # noqa: PLC0415++ CacheArtifactManager._ensure_cache_artifacts_registered()+ out = AppendingByteSerializer(serialize_fn = _serialize_single_cache)+ for data in bundles:+ out.extend(+ AppendingByteSerializer.to_list(data, deserialize_fn = _deserialize_single_cache)+ )+ return _one_group_per_type(out.to_bytes(), logger)+ except Exception as exc: # noqa: BLE001+ _warn(+ logger,+ f"compile-cache: could not merge earlier collections ({exc}); saving this one alone",+ )+ return _one_group_per_type(bundles[-1], logger) if bundles else None++def _one_group_per_type(data: bytes, logger: Any) -> bytes:"""One ``(type, artifacts)`` group per type: torch appends a group per save and loads them with ``dict(...)``,so a re-saved bundle kept only the newest group of each type."""@@ -896,11 +1099,10 @@ def _write_bundle(ctx: CacheContext, logger: Any) -> bool:except Exception as exc: # noqa: BLE001_warn(logger, f"compile-cache: save_cache_artifacts failed: {exc}")return False- if not result or result[0] is None:+ data = _with_reused_collections(ctx, result[0] if result else None, logger)+ if data is None:_warn(logger, "compile-cache: nothing to save (empty artifacts)")return False-- data = _one_group_per_type(result[0], logger)try:ctx.dir.mkdir(parents = True, exist_ok = True)digest = hashlib.sha256(data).hexdigest()diff --git a/studio/backend/tests/test_diffusion_compile_cache_collector.py b/studio/backend/tests/test_diffusion_compile_cache_collector.pynew file mode 100644index 00000000000..1e878004abb--- /dev/null+++ b/studio/backend/tests/test_diffusion_compile_cache_collector.py@@ -0,0 +1,243 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Each model's compile bundle holds its own artifacts, not every artifact torch collected since startup.++Real ``torch.compiler`` save / load and torch's real artifact collector, fed synthetic Inductor artifacts."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")+_cache = pytest.importorskip("torch.compiler._cache")++from core.inference import diffusion_compile_cache as cc # noqa: E402++ISOLATE_ENV = "UNSLOTH_DIFFUSION_COMPILE_CACHE_ISOLATE"+_PAYLOAD = 64 << 10+++def collector(monkeypatch, tmp_path):+ from torch._inductor.codecache import InductorCacheArtifact # noqa: F401 - registers the type++ monkeypatch.setenv(cc._ENV_MODE, "on")+ monkeypatch.setenv(cc._ENV_SYNC, "1")+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))+ monkeypatch.delenv(ISOLATE_ENV, raising = False)+ monkeypatch.setattr(cc, "_live_dirs", {})+ # synthetic artifacts never populate a real inductor cache+ monkeypatch.setattr(_cache.CacheArtifact, "populate_cache", lambda self: None)+ with _cache.CacheArtifactManager.with_fresh_cache():+ yield+++def _record(name: str) -> None:+ from torch._inductor.codecache import InductorCacheArtifact+ _cache.CacheArtifactManager.record_artifact(+ InductorCacheArtifact.type(), f"{name}-graph", bytes([ord(name[0])]) * _PAYLOAD+ )+++def _lifecycle(name: str, restore: bool = True):+ ctx = cc.begin(+ transformer = types.SimpleNamespace(_repeated_blocks = [f"{name}Block"]),+ family = name,+ dtype = "torch.bfloat16",+ quant = None,+ attention_backend = "native",+ compile_kwargs = {"fullgraph": True},+ )+ assert ctx is not None and not ctx.hit+ _record(name)+ assert cc.save(ctx)+ if restore:+ cc.restore(ctx)+ return ctx+++def _keys(ctx) -> set:+ artifacts = _cache.CacheArtifactManager.deserialize(ctx.bundle.read_bytes())+ return {a.key for group in artifacts.values() for a in group}+++def test_each_models_bundle_holds_only_its_own_artifacts(collector):+ ctxs = [_lifecycle(name) for name in ("alpha", "beta", "gamma", "delta")]+ sizes = [c.bundle.stat().st_size for c in ctxs]+ assert (+ max(sizes) - min(sizes) < 1024+ ) # 1 artifact each; torch's collector alone gives 1, 2, 3, 4+ assert [_keys(c) for c in ctxs] == [{f"{n}-graph"} for n in ("alpha", "beta", "gamma", "delta")]+++def test_a_saved_bundle_still_loads_into_a_fresh_collector(collector):+ first = _lifecycle("alpha")+ _lifecycle("beta")+ with _cache.CacheArtifactManager.with_fresh_cache():+ assert torch.compiler.load_cache_artifacts(first.bundle.read_bytes()) is not None+++def test_the_isolation_switch_keeps_torchs_accumulating_collector(collector, monkeypatch):+ monkeypatch.setenv(ISOLATE_ENV, "0")+ assert cc._ENV_ISOLATE == ISOLATE_ENV+ ctxs = [_lifecycle(name) for name in ("alpha", "beta", "gamma")]+ sizes = [c.bundle.stat().st_size for c in ctxs]+ assert sizes[1] > sizes[0] + _PAYLOAD // 2 and sizes[2] > sizes[1] + _PAYLOAD // 2+ assert _keys(ctxs[2]) == {"alpha-graph", "beta-graph", "gamma-graph"}+++def test_a_bundle_still_live_keeps_the_collector(collector):+ # a reload overlapping its predecessor's teardown: the live model's pending artifacts are not dropped+ first = _lifecycle("alpha", restore = False)+ second = _lifecycle("beta")+ assert _keys(second) == {"alpha-graph", "beta-graph"}+ cc.restore(first)+++def test_a_running_compile_is_never_waited_for(collector, monkeypatch):+ import threading++ lock = threading.Lock()+ lock.acquire()+ monkeypatch.setattr(cc, "_compile_lock", lambda: lock)+ _record("pending")+ assert cc._fresh_collector(None) is False+ assert _cache.CacheArtifactManager.need_serialize()+ lock.release()+ assert cc._fresh_collector(None) is True+ assert not _cache.CacheArtifactManager.need_serialize()+++# Real CPU Inductor: a later model of the same classes runs earlier graphs from memory and records nothing.+++class _Net(torch.nn.Module):+ def __init__(+ self,+ value: float,+ graph_break: bool = False,+ ):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.full((16, 16), value))+ self.graph_break = graph_break++ def forward(self, x):+ x = torch.sin(x @ self.weight)+ if self.graph_break:+ torch._dynamo.graph_break()+ return torch.cos(x) * 2+++class _Other(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.full((16, 16), 0.5))++ def forward(self, x):+ return torch.tanh(x @ self.weight) + 1+++def real_compile(monkeypatch, tmp_path):+ monkeypatch.setenv(cc._ENV_MODE, "on")+ monkeypatch.setenv(cc._ENV_SYNC, "1")+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path / "bundles"))+ monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path / "inductor"))+ monkeypatch.delenv(ISOLATE_ENV, raising = False)+ monkeypatch.setattr(cc, "_live_dirs", {})+ monkeypatch.setattr(cc, "_collections", [], raising = False)+ monkeypatch.setattr(cc, "_collecting_since", None, raising = False)+ monkeypatch.setattr(cc, "_collecting_forwards", set(), raising = False)+ torch._dynamo.reset()+ try:+ with _cache.CacheArtifactManager.with_fresh_cache(), torch.no_grad():+ try:+ torch.compile(lambda x: x.sin() + 1, backend = "inductor")(torch.randn(2))+ except Exception as exc: # noqa: BLE001 - no C++ toolchain for CPU Inductor here+ pytest.skip(f"CPU Inductor cannot compile here: {exc}")+ torch._dynamo.reset()+ _cache.CacheArtifactManager.clear()+ yield+ finally:+ torch._dynamo.reset()+++def _compiled_lifecycle(+ name: str,+ net,+ shapes = ((4, 16),),+ reset_dynamo = False,+):+ ctx = cc.begin(+ family = name,+ transformer = net,+ dtype = "torch.float32",+ quant = None,+ attention_backend = "native",+ compile_kwargs = {"fullgraph": False, "dynamic": False},+ )+ assert ctx is not None+ if reset_dynamo:+ torch._dynamo.reset()+ compiled = torch.compile(net, dynamic = False)+ for shape in shapes:+ x = torch.randn(*shape)+ torch.testing.assert_close(compiled(x), net(x))+ saved = cc.save(ctx)+ cc.restore(ctx)+ return ctx, saved+++def _inductor_keys(ctx) -> set:+ artifacts = _cache.CacheArtifactManager.deserialize(ctx.bundle.read_bytes())+ return {a.key for a in artifacts.get("inductor", [])}+++def _unique_graphs() -> int:+ from torch._dynamo.utils import counters+ return counters["stats"]["unique_graphs"]++[email protected]("graph_break", [False, True], ids = ["fullgraph", "graph_break"])+def test_a_model_reusing_an_earlier_models_compiled_graph_still_gets_a_bundle(+ real_compile, graph_break+):+ first, first_saved = _compiled_lifecycle("model_a", _Net(1.0, graph_break))+ graphs = _unique_graphs()+ second, second_saved = _compiled_lifecycle("model_b", _Net(2.0, graph_break))+ assert first.key != second.key+ # model_b compiled nothing: it ran model_a's graphs from Dynamo's in-memory cache+ assert _unique_graphs() == graphs+ assert first_saved and second_saved+ assert _inductor_keys(second) == _inductor_keys(first) != set()+ with _cache.CacheArtifactManager.with_fresh_cache():+ assert torch.compiler.load_cache_artifacts(second.bundle.read_bytes()) is not None+++def test_an_unrelated_model_still_gets_only_its_own_artifacts(real_compile):+ first, _ = _compiled_lifecycle("model_a", _Net(1.0))+ other, saved = _compiled_lifecycle("other", _Other())+ assert saved+ assert _inductor_keys(other) and not _inductor_keys(other) & _inductor_keys(first)+++def test_a_reload_that_compiles_a_new_shape_keeps_the_shape_it_reused(real_compile):+ net = _Net(1.0)+ first, _ = _compiled_lifecycle("model_a", net, shapes = ((4, 16),))+ reloaded, saved = _compiled_lifecycle("model_a", net, shapes = ((4, 16), (8, 16)))+ assert reloaded.key == first.key and saved+ keys = _inductor_keys(reloaded)+ assert _inductor_keys(first) < keys+++def test_a_graph_recompiled_at_an_unchanged_entry_count_is_still_kept(real_compile):+ first, _ = _compiled_lifecycle("model_a", _Net(1.0))+ # reset + recompile at another shape: still one entry, but a new graph+ second, _ = _compiled_lifecycle("model_b", _Net(2.0), shapes = ((8, 16),), reset_dynamo = True)+ third, saved = _compiled_lifecycle("model_c", _Net(3.0), shapes = ((8, 16),))+ assert saved and _inductor_keys(second) - _inductor_keys(first)+ # model_a's graph left Dynamo, so its artifacts are not carried+ assert _inductor_keys(third) == _inductor_keys(second)