Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: each model's compile bundle holds only its own artifacts (#17110)

Daniel Hand4f72beff577
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index baacc49d799..af60fb6b96f 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -29,6 +29,8 @@ _ENV_DIR = "UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR"
_ENV_SAVE = "UNSLOTH_DIFFUSION_COMPILE_CACHE_SAVE"
_ENV_SYNC = "UNSLOTH_DIFFUSION_COMPILE_CACHE_SYNC"
_ENV_MAX_GB = "UNSLOTH_DIFFUSION_COMPILE_CACHE_MAX_GB"
+# "0": keep torch's process-wide artifact collector, so each bundle also carries earlier models' artifacts.
+_ENV_ISOLATE = "UNSLOTH_DIFFUSION_COMPILE_CACHE_ISOLATE"
_DEFAULT_MAX_GB = 20.0
_LEGACY_ROOT = Path.home() / ".cache" / "unsloth" / "diffusion_compile_cache"
@@ -509,6 +511,7 @@ class CacheContext:
# Recompiled artifacts often hash the same, so overwrite instead of the exists() shortcut.
rejected_bundle: Optional[str] = None
last_touch: float = 0.0
+ forwards: frozenset = frozenset()
def begin(
@@ -560,14 +563,18 @@ def begin(
env_fp = env_fp,
model_fp = model_fp,
mode = mode,
+ forwards = _module_forwards(transformer),
)
# Same reason as restore(): the previous load's save reads the inductor dir that is about to be repointed.
- if not wait_for_saves():
+ drained = wait_for_saves()
+ if not drained:
_warn(
logger,
f"compile-cache: background save still running after {_SAVE_JOIN_TIMEOUT:.0f}s; continuing",
)
+ else:
+ _fresh_collector(logger, ctx.forwards)
try:
cdir.mkdir(parents = True, exist_ok = True)
@@ -835,6 +842,202 @@ def _compile_lock() -> Any:
return contextlib.nullcontext()
+def _isolation_off() -> bool:
+ return (os.environ.get(_ENV_ISOLATE) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _module_forwards(transformer: Any) -> frozenset:
+ """``forward`` codes of ``transformer``'s module classes; Dynamo shares compiled code per code object."""
+ codes = set()
+ try:
+ modules = transformer.modules() if callable(getattr(transformer, "modules", None)) else ()
+ for module in modules:
+ code = getattr(getattr(type(module), "forward", None), "__code__", None)
+ if code is not None:
+ codes.add(code)
+ except Exception: # noqa: BLE001
+ pass
+ return frozenset(codes)
+
+
+def _dynamo_entries() -> Optional[dict[int, tuple[Any, tuple]]]:
+ """``id(code) -> (code, each cache entry's compiled code)``; None when torch does not expose them."""
+ try:
+ from torch._dynamo import convert_frame # noqa: PLC0415
+ from torch._dynamo.eval_frame import _debug_get_cache_entry_list # noqa: PLC0415
+
+ entries = {}
+ for ref in list(convert_frame.input_codes.seen):
+ code = ref()
+ if code is None:
+ continue
+ compiled = tuple(entry.code for entry in _debug_get_cache_entry_list(code))
+ if compiled:
+ entries[id(code)] = (code, compiled)
+ return entries
+ except Exception: # noqa: BLE001
+ return None
+
+
+_RESUME_RE = re.compile(r"torch_dynamo_resume_in_(.+)_at_\d+")
+
+
+def _serves(code: Any, forwards: Any) -> bool:
+ """``code`` is one of ``forwards`` or a graph-break resume function of one (matched by name and file)."""
+ if code in forwards:
+ return True
+ name = code.co_name
+ resumed = False
+ while (match := _RESUME_RE.fullmatch(name)) is not None:
+ name, resumed = match.group(1), True
+ return resumed and any(
+ f.co_name == name and f.co_filename == code.co_filename for f in forwards
+ )
+
+
+def _still_compiled(collection: "_Collection") -> bool:
+ """Dynamo still holds an entry compiled during ``collection``."""
+ try:
+ from torch._dynamo.eval_frame import _debug_get_cache_entry_list # noqa: PLC0415
+ ids = {id(c) for c in collection.compiled}
+ return any(
+ id(entry.code) in ids
+ for code in collection.codes
+ for entry in _debug_get_cache_entry_list(code)
+ )
+ except Exception: # noqa: BLE001 - cannot tell, so keep it
+ return True
+
+
+class _Collection:
+ """Artifacts torch collected between two resets, kept while Dynamo can still run what was compiled then."""
+
+ data: bytes
+ codes: frozenset
+ compiled: tuple
+ # some compiled code is no forward of the models served, so any model may run it
+ any_model: bool
+
+
+_collections_lock = threading.Lock()
+_collections: list[_Collection] = []
+# the open collection: Dynamo's entries at its start, and the forwards it serves
+_collecting_since: Optional[dict[int, tuple[Any, tuple]]] = None
+_collecting_forwards: set = set()
+# past this the collector just keeps accumulating, as torch does
+_COLLECTIONS_MAX_BYTES = 2 << 30
+
+
+def _fresh_collector(logger: Any, forwards: frozenset = frozenset()) -> bool:
+ """Reset torch's artifact collector so this model's bundle holds its own artifacts. A Dynamo in-memory hit
+ records nothing, so the previous collection is kept for ``save`` to add back; skipped while another bundle is live."""
+ global _collecting_since
+ with _collections_lock:
+ _collecting_forwards.update(forwards)
+ if _isolation_off():
+ return False
+ with _live_lock:
+ if _live_dirs:
+ return False
+ try:
+ import torch # noqa: PLC0415
+ from torch.compiler._cache import CacheArtifactManager # noqa: PLC0415
+ from torch.utils._appending_byte_serializer import AppendingByteSerializer # noqa: F401, PLC0415
+ except ImportError:
+ # torch 2.7 bundles are a flat list that cannot be merged
+ return False
+ lock = _compile_lock()
+ acquire = getattr(lock, "acquire", None)
+ # never wait: a running compile's artifacts are not this model's to drop
+ if acquire is not None and not acquire(blocking = False):
+ return False
+ try:
+ entries = _dynamo_entries()
+ if entries is None:
+ return False
+ with _collections_lock:
+ since = _collecting_since if _collecting_since is not None else {}
+ # by identity: a reset then recompile keeps the count; ``since`` keeps the ids alive
+ new = {}
+ for key, (code, compiled) in entries.items():
+ before = {id(c) for c in since.get(key, (None, ()))[1]}
+ fresh = tuple(c for c in compiled if id(c) not in before)
+ if fresh:
+ new[code] = fresh
+ grown = frozenset(new)
+ _collections[:] = [c for c in _collections if _still_compiled(c)]
+ if grown:
+ result = torch.compiler.save_cache_artifacts()
+ data = result[0] if result else None
+ if data:
+ if sum(len(c.data) for c in _collections) + len(data) > _COLLECTIONS_MAX_BYTES:
+ return False
+ _collections.append(
+ _Collection(
+ data = data,
+ codes = grown,
+ compiled = tuple(c for fresh in new.values() for c in fresh),
+ any_model = not all(
+ _serves(code, _collecting_forwards) for code in grown
+ ),
+ )
+ )
+ # no new entry: nothing from this collection runs without recompiling, which records again
+ CacheArtifactManager.clear()
+ _collecting_since = entries
+ _collecting_forwards.clear()
+ _collecting_forwards.update(forwards)
+ return True
+ except Exception as exc: # noqa: BLE001 - keep torch's collector as it is
+ _warn(logger, f"compile-cache: could not reset the artifact collector: {exc}")
+ return False
+ finally:
+ if acquire is not None:
+ lock.release()
+
+
+def _with_reused_collections(
+ ctx: CacheContext, data: Optional[bytes], logger: Any
+) -> Optional[bytes]:
+ """``data`` plus each kept collection ``ctx``'s model may run, merged; None when there is nothing."""
+ with _collections_lock:
+ reused = [
+ c.data
+ for c in _collections
+ if (c.any_model or any(_serves(code, ctx.forwards) for code in c.codes))
+ and _still_compiled(c)
+ ]
+ if not reused:
+ return None if data is None else _one_group_per_type(data, logger)
+ return _merge_groups(reused + ([data] if data else []), logger)
+
+
+def _merge_groups(bundles: list[bytes], logger: Any) -> Optional[bytes]:
+ """Serialized collections, oldest first, as one bundle."""
+ try:
+ from torch.compiler._cache import ( # noqa: PLC0415
+ CacheArtifactManager,
+ _deserialize_single_cache,
+ _serialize_single_cache,
+ )
+ from torch.utils._appending_byte_serializer import AppendingByteSerializer # noqa: PLC0415
+
+ CacheArtifactManager._ensure_cache_artifacts_registered()
+ out = AppendingByteSerializer(serialize_fn = _serialize_single_cache)
+ for data in bundles:
+ out.extend(
+ AppendingByteSerializer.to_list(data, deserialize_fn = _deserialize_single_cache)
+ )
+ return _one_group_per_type(out.to_bytes(), logger)
+ except Exception as exc: # noqa: BLE001
+ _warn(
+ logger,
+ f"compile-cache: could not merge earlier collections ({exc}); saving this one alone",
+ )
+ return _one_group_per_type(bundles[-1], logger) if bundles else None
+
+
def _one_group_per_type(data: bytes, logger: Any) -> bytes:
"""One ``(type, artifacts)`` group per type: torch appends a group per save and loads them with ``dict(...)``,
so a re-saved bundle kept only the newest group of each type."""
@@ -896,11 +1099,10 @@ def _write_bundle(ctx: CacheContext, logger: Any) -> bool:
except Exception as exc: # noqa: BLE001
_warn(logger, f"compile-cache: save_cache_artifacts failed: {exc}")
return False
- if not result or result[0] is None:
+ data = _with_reused_collections(ctx, result[0] if result else None, logger)
+ if data is None:
_warn(logger, "compile-cache: nothing to save (empty artifacts)")
return False
-
- data = _one_group_per_type(result[0], logger)
try:
ctx.dir.mkdir(parents = True, exist_ok = True)
digest = hashlib.sha256(data).hexdigest()
diff --git a/studio/backend/tests/test_diffusion_compile_cache_collector.py b/studio/backend/tests/test_diffusion_compile_cache_collector.py
new file mode 100644
index 00000000000..1e878004abb
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_compile_cache_collector.py
@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Each model's compile bundle holds its own artifacts, not every artifact torch collected since startup.
+
+Real ``torch.compiler`` save / load and torch's real artifact collector, fed synthetic Inductor artifacts."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+_cache = pytest.importorskip("torch.compiler._cache")
+
+from core.inference import diffusion_compile_cache as cc # noqa: E402
+
+ISOLATE_ENV = "UNSLOTH_DIFFUSION_COMPILE_CACHE_ISOLATE"
+_PAYLOAD = 64 << 10
+
+
+def collector(monkeypatch, tmp_path):
+ from torch._inductor.codecache import InductorCacheArtifact # noqa: F401 - registers the type
+
+ monkeypatch.setenv(cc._ENV_MODE, "on")
+ monkeypatch.setenv(cc._ENV_SYNC, "1")
+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path))
+ monkeypatch.delenv(ISOLATE_ENV, raising = False)
+ monkeypatch.setattr(cc, "_live_dirs", {})
+ # synthetic artifacts never populate a real inductor cache
+ monkeypatch.setattr(_cache.CacheArtifact, "populate_cache", lambda self: None)
+ with _cache.CacheArtifactManager.with_fresh_cache():
+ yield
+
+
+def _record(name: str) -> None:
+ from torch._inductor.codecache import InductorCacheArtifact
+ _cache.CacheArtifactManager.record_artifact(
+ InductorCacheArtifact.type(), f"{name}-graph", bytes([ord(name[0])]) * _PAYLOAD
+ )
+
+
+def _lifecycle(name: str, restore: bool = True):
+ ctx = cc.begin(
+ transformer = types.SimpleNamespace(_repeated_blocks = [f"{name}Block"]),
+ family = name,
+ dtype = "torch.bfloat16",
+ quant = None,
+ attention_backend = "native",
+ compile_kwargs = {"fullgraph": True},
+ )
+ assert ctx is not None and not ctx.hit
+ _record(name)
+ assert cc.save(ctx)
+ if restore:
+ cc.restore(ctx)
+ return ctx
+
+
+def _keys(ctx) -> set:
+ artifacts = _cache.CacheArtifactManager.deserialize(ctx.bundle.read_bytes())
+ return {a.key for group in artifacts.values() for a in group}
+
+
+def test_each_models_bundle_holds_only_its_own_artifacts(collector):
+ ctxs = [_lifecycle(name) for name in ("alpha", "beta", "gamma", "delta")]
+ sizes = [c.bundle.stat().st_size for c in ctxs]
+ assert (
+ max(sizes) - min(sizes) < 1024
+ ) # 1 artifact each; torch's collector alone gives 1, 2, 3, 4
+ assert [_keys(c) for c in ctxs] == [{f"{n}-graph"} for n in ("alpha", "beta", "gamma", "delta")]
+
+
+def test_a_saved_bundle_still_loads_into_a_fresh_collector(collector):
+ first = _lifecycle("alpha")
+ _lifecycle("beta")
+ with _cache.CacheArtifactManager.with_fresh_cache():
+ assert torch.compiler.load_cache_artifacts(first.bundle.read_bytes()) is not None
+
+
+def test_the_isolation_switch_keeps_torchs_accumulating_collector(collector, monkeypatch):
+ monkeypatch.setenv(ISOLATE_ENV, "0")
+ assert cc._ENV_ISOLATE == ISOLATE_ENV
+ ctxs = [_lifecycle(name) for name in ("alpha", "beta", "gamma")]
+ sizes = [c.bundle.stat().st_size for c in ctxs]
+ assert sizes[1] > sizes[0] + _PAYLOAD // 2 and sizes[2] > sizes[1] + _PAYLOAD // 2
+ assert _keys(ctxs[2]) == {"alpha-graph", "beta-graph", "gamma-graph"}
+
+
+def test_a_bundle_still_live_keeps_the_collector(collector):
+ # a reload overlapping its predecessor's teardown: the live model's pending artifacts are not dropped
+ first = _lifecycle("alpha", restore = False)
+ second = _lifecycle("beta")
+ assert _keys(second) == {"alpha-graph", "beta-graph"}
+ cc.restore(first)
+
+
+def test_a_running_compile_is_never_waited_for(collector, monkeypatch):
+ import threading
+
+ lock = threading.Lock()
+ lock.acquire()
+ monkeypatch.setattr(cc, "_compile_lock", lambda: lock)
+ _record("pending")
+ assert cc._fresh_collector(None) is False
+ assert _cache.CacheArtifactManager.need_serialize()
+ lock.release()
+ assert cc._fresh_collector(None) is True
+ assert not _cache.CacheArtifactManager.need_serialize()
+
+
+# Real CPU Inductor: a later model of the same classes runs earlier graphs from memory and records nothing.
+
+
+class _Net(torch.nn.Module):
+ def __init__(
+ self,
+ value: float,
+ graph_break: bool = False,
+ ):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.full((16, 16), value))
+ self.graph_break = graph_break
+
+ def forward(self, x):
+ x = torch.sin(x @ self.weight)
+ if self.graph_break:
+ torch._dynamo.graph_break()
+ return torch.cos(x) * 2
+
+
+class _Other(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.full((16, 16), 0.5))
+
+ def forward(self, x):
+ return torch.tanh(x @ self.weight) + 1
+
+
+def real_compile(monkeypatch, tmp_path):
+ monkeypatch.setenv(cc._ENV_MODE, "on")
+ monkeypatch.setenv(cc._ENV_SYNC, "1")
+ monkeypatch.setenv(cc._ENV_DIR, str(tmp_path / "bundles"))
+ monkeypatch.setenv("TORCHINDUCTOR_CACHE_DIR", str(tmp_path / "inductor"))
+ monkeypatch.delenv(ISOLATE_ENV, raising = False)
+ monkeypatch.setattr(cc, "_live_dirs", {})
+ monkeypatch.setattr(cc, "_collections", [], raising = False)
+ monkeypatch.setattr(cc, "_collecting_since", None, raising = False)
+ monkeypatch.setattr(cc, "_collecting_forwards", set(), raising = False)
+ torch._dynamo.reset()
+ try:
+ with _cache.CacheArtifactManager.with_fresh_cache(), torch.no_grad():
+ try:
+ torch.compile(lambda x: x.sin() + 1, backend = "inductor")(torch.randn(2))
+ except Exception as exc: # noqa: BLE001 - no C++ toolchain for CPU Inductor here
+ pytest.skip(f"CPU Inductor cannot compile here: {exc}")
+ torch._dynamo.reset()
+ _cache.CacheArtifactManager.clear()
+ yield
+ finally:
+ torch._dynamo.reset()
+
+
+def _compiled_lifecycle(
+ name: str,
+ net,
+ shapes = ((4, 16),),
+ reset_dynamo = False,
+):
+ ctx = cc.begin(
+ family = name,
+ transformer = net,
+ dtype = "torch.float32",
+ quant = None,
+ attention_backend = "native",
+ compile_kwargs = {"fullgraph": False, "dynamic": False},
+ )
+ assert ctx is not None
+ if reset_dynamo:
+ torch._dynamo.reset()
+ compiled = torch.compile(net, dynamic = False)
+ for shape in shapes:
+ x = torch.randn(*shape)
+ torch.testing.assert_close(compiled(x), net(x))
+ saved = cc.save(ctx)
+ cc.restore(ctx)
+ return ctx, saved
+
+
+def _inductor_keys(ctx) -> set:
+ artifacts = _cache.CacheArtifactManager.deserialize(ctx.bundle.read_bytes())
+ return {a.key for a in artifacts.get("inductor", [])}
+
+
+def _unique_graphs() -> int:
+ from torch._dynamo.utils import counters
+ return counters["stats"]["unique_graphs"]
+
+
[email protected]("graph_break", [False, True], ids = ["fullgraph", "graph_break"])
+def test_a_model_reusing_an_earlier_models_compiled_graph_still_gets_a_bundle(
+ real_compile, graph_break
+):
+ first, first_saved = _compiled_lifecycle("model_a", _Net(1.0, graph_break))
+ graphs = _unique_graphs()
+ second, second_saved = _compiled_lifecycle("model_b", _Net(2.0, graph_break))
+ assert first.key != second.key
+ # model_b compiled nothing: it ran model_a's graphs from Dynamo's in-memory cache
+ assert _unique_graphs() == graphs
+ assert first_saved and second_saved
+ assert _inductor_keys(second) == _inductor_keys(first) != set()
+ with _cache.CacheArtifactManager.with_fresh_cache():
+ assert torch.compiler.load_cache_artifacts(second.bundle.read_bytes()) is not None
+
+
+def test_an_unrelated_model_still_gets_only_its_own_artifacts(real_compile):
+ first, _ = _compiled_lifecycle("model_a", _Net(1.0))
+ other, saved = _compiled_lifecycle("other", _Other())
+ assert saved
+ assert _inductor_keys(other) and not _inductor_keys(other) & _inductor_keys(first)
+
+
+def test_a_reload_that_compiles_a_new_shape_keeps_the_shape_it_reused(real_compile):
+ net = _Net(1.0)
+ first, _ = _compiled_lifecycle("model_a", net, shapes = ((4, 16),))
+ reloaded, saved = _compiled_lifecycle("model_a", net, shapes = ((4, 16), (8, 16)))
+ assert reloaded.key == first.key and saved
+ keys = _inductor_keys(reloaded)
+ assert _inductor_keys(first) < keys
+
+
+def test_a_graph_recompiled_at_an_unchanged_entry_count_is_still_kept(real_compile):
+ first, _ = _compiled_lifecycle("model_a", _Net(1.0))
+ # reset + recompile at another shape: still one entry, but a new graph
+ second, _ = _compiled_lifecycle("model_b", _Net(2.0), shapes = ((8, 16),), reset_dynamo = True)
+ third, saved = _compiled_lifecycle("model_c", _Net(3.0), shapes = ((8, 16),))
+ assert saved and _inductor_keys(second) - _inductor_keys(first)
+ # model_a's graph left Dynamo, so its artifacts are not carried
+ assert _inductor_keys(third) == _inductor_keys(second)