Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
CFG-parallel: treat a changed Python number, storage offset or autocast as a cold call (#14614)
Daniel Hana07c510b6861
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex d5bb17d208b..c4828900b03 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -1717,21 +1717,38 @@ def _mirror_hook_state(pairs: list) -> tuple:return state+def _autocast_state(torch: Any) -> tuple:+ try:+ return (+ torch.is_autocast_cache_enabled(),+ *(+ (dev, torch.get_autocast_dtype(dev))+ if torch.is_autocast_enabled(dev)+ else (dev, None)+ for dev in ("cuda", "cpu")+ ),+ )+ except Exception: # noqa: BLE001 - torch without per-device autocast queries+ return ()++def _call_signature(args: tuple, kwargs: dict) -> tuple:- """What a compiled forward's guards mostly key on: tensor layout, flags and modes (never per-step scalars)."""+ """What a compiled forward's guards key on, numbers by value: 0 / 1 and ``dynamic=False`` compiles stay specialised."""torch = _torch()sig: list = []for path, leaf in _leaves((args, kwargs)):if isinstance(leaf, torch.Tensor):try:stride = tuple(leaf.stride())+ offset = leaf.storage_offset()except Exception: # noqa: BLE001 - layouts without strides- stride = None+ stride = offset = Nonesig.append((path,tuple(leaf.shape),stride,+ offset,leaf.dtype,leaf.requires_grad,leaf.is_inference(),@@ -1739,9 +1756,16 @@ def _call_signature(args: tuple, kwargs: dict) -> tuple:)elif leaf is None or isinstance(leaf, (bool, str)):sig.append((path, leaf))+ elif isinstance(leaf, (int, float)):+ sig.append((path, type(leaf), leaf))else:sig.append((path, type(leaf)))- return tuple(sig), torch.is_inference_mode_enabled(), torch.is_grad_enabled()+ return (+ tuple(sig),+ torch.is_inference_mode_enabled(),+ torch.is_grad_enabled(),+ _autocast_state(torch),+ )def _note_warm(seen: set, sig: tuple) -> None:diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.pyindex 26233d9979f..4a322f961b2 100644--- a/studio/backend/tests/test_diffusion_multigpu.py+++ b/studio/backend/tests/test_diffusion_multigpu.py@@ -2366,3 +2366,51 @@ def test_replica_blocks_get_their_own_compile_guard():guard = getattr(wrapped, "_unsloth_compile_guard", None)assert guard is not None and guard is not source_guardassert replica.__dict__.get("_unsloth_compile_guard") is guard++[email protected]("dynamic", [None, False])+def test_a_warm_signature_never_compiles_on_a_changed_number(dynamic):+ # A warm call runs beside the other side, so it must never compile (FX tracing is process-wide up to 2.13).+ compiles = []++ def backend(gm, _example_inputs):+ compiles.append(1)+ return gm.forward++ @torch.compile(backend = backend, dynamic = dynamic)+ def fn(x, scale, k, kw):+ return x * scale + k + kw["s"]++ x = torch.randn(4)+ warm: set = set()+ calls = [+ (1.0, 1, 0.5),+ (2.0, 1, 0.5),+ (3.0, 1, 0.5),+ (3.0, 2, 0.5),+ (3.0, 3, 0.5),+ (4.5, 7, 0.25),+ (4.5, 0, 0.25),+ (4.5, 1, 0.25),+ (4.5, 7, 0.25),+ (5.5, 1, 0.25),+ ]+ for scale, k, s in calls:+ sig = mg._call_signature((x, scale, k), {"kw": {"s": s}})+ before = len(compiles)+ fn(x, scale, k, {"s": s})+ if sig in warm:+ assert len(compiles) == before, (scale, k, s)+ mg._note_warm(warm, sig)+ torch._dynamo.reset()+++def test_warm_signature_keys_on_storage_offset_and_autocast():+ base = torch.zeros(8)+ a = mg._call_signature((base[:4],), {})+ assert mg._call_signature((base[4:],), {}) != a+ with torch.autocast("cpu", dtype = torch.bfloat16):+ assert mg._call_signature((base[:4],), {}) != a+ with torch.autocast("cpu", enabled = False, cache_enabled = False):+ assert mg._call_signature((base[:4],), {}) != a+ assert mg._call_signature((base[:4],), {}) == a