Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio CFG-parallel: treat an NVFP4 precision-state flip as a cold call so the two branches never compile at once (#14854)
Daniel Han6d2e6bf92aed
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex c5d1041602d..e0371131d5c 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -2017,9 +2017,8 @@ def _copy_module(module: Any, device: Any) -> Any:memo[id(hook)] = _DROP# Per-step precision state (the NVFP4 W4A16 protect schedule) is advanced by the source's scheduler: the copy# must read the same controller, or the two CFG branches of one step run different GEMMs.- for value in sub.__dict__.values():- if type(value).__name__ in _SHARED_STATE_TYPES:- memo[id(value)] = value+ for value in _shared_state_values(sub):+ memo[id(value)] = valueguard = module.__dict__.get("_unsloth_compile_guard")if guard is not None:memo[id(guard)] = _DROP@@ -2094,6 +2093,10 @@ _SHARED_STATE_TYPES = frozenset({"NVFP4StepController"})_SOURCE_ONLY_ATTRS = frozenset({"_unsloth_cond_cache", "_unsloth_rocm_flash_replicas"})+def _shared_state_values(sub: Any) -> list:+ return [value for value in sub.__dict__.values() if type(value).__name__ in _SHARED_STATE_TYPES]++def _rewrap_compiled(source: Any, replica: Any) -> None:"""Re-apply the wrappers Studio's compile path puts around the source's compiled blocks (``dst.compile`` replacedthem): the compile guard (eager fallback on a compile-time failure, the background-compile eager gate, the compile@@ -2679,6 +2682,25 @@ def _call_signature(args: tuple, kwargs: dict) -> tuple:)+def _shared_states(pairs: list) -> tuple:+ found: dict = {}+ for pair in pairs:+ for sub in pair:+ for value in _shared_state_values(sub):+ found.setdefault(id(value), value)+ return tuple(found.values())+++def _shared_state_key(states: tuple) -> tuple:+ key = []+ for state in states:+ try:+ key.append(state.compile_key())+ except Exception: # noqa: BLE001 - an unreadable state is never warm+ key.append(object())+ return tuple(key)++def _note_warm(seen: set, sig: tuple) -> None:if len(seen) >= 256:seen.clear()@@ -2779,6 +2801,8 @@ class CfgParallelRouter:self.probe_index: dict = {}self.probed_this_render = Falseself.pairs: list = []+ # Shared per-step states' compile guards join every warm-call signature.+ self.shared_states: tuple = ()# A call new to either side may compile (FX tracing is process-wide up to torch 2.13): it never overlaps the other side.self.warm_primary: set = set()self.warm_replica: set = set()@@ -2822,6 +2846,7 @@ class CfgParallelRouter:self.kv_mirrors.clear()self.kv_declined.clear()self.pairs = []+ self.shared_states = ()self.warm_primary.clear()self.warm_replica.clear()self.prefetched.clear()@@ -2865,12 +2890,16 @@ class CfgParallelRouter:self.exclusion_refused = Falsereturn False+ def _signature(self, args: tuple, kwargs: dict) -> tuple:+ """``_call_signature`` plus guarded model state: an NVFP4 protect flip recompiles under seen args."""+ return (*_call_signature(args, kwargs), _shared_state_key(self.shared_states))+def _primary(self, *args: Any, **kwargs: Any) -> Any:inner = self.innerreturn (inner if inner is not None else self.orig_forward)(*args, **kwargs)def _primary_noting(self, sig: Optional[tuple], args: tuple, kwargs: dict) -> Any:- sig = _call_signature(args, kwargs) if sig is None else sig+ sig = self._signature(args, kwargs) if sig is None else sigjob = self._pair_jobif sig not in self.warm_primary and job is not None:job["event"].wait() # may compile: not while the replica runs@@ -2909,7 +2938,7 @@ class CfgParallelRouter:# Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after# it would wait for it on the source stream and serialise the two branches.self._prefetch(args, kwargs)- sig = _call_signature(args, kwargs)+ sig = self._signature(args, kwargs)self._launch_prediction(latent, sig)if (self.branch == 0@@ -3060,7 +3089,7 @@ class CfgParallelRouter:return Noneself.prefetched.clear()hook_state = _mirror_hook_state(self.pairs)- sig = _call_signature(args, kwargs)+ sig = self._signature(args, kwargs)try:r_args = self._to_replica(args)r_kwargs = self._to_replica(kwargs)@@ -3154,7 +3183,7 @@ class CfgParallelRouter:returnif (sig not in self.warm_primary- or _call_signature(p_args, p_kwargs) not in self.warm_replica+ or self._signature(p_args, p_kwargs) not in self.warm_replica):return # a side may compile: the odd branch runs unpredicted, after the primary onetry:@@ -3304,6 +3333,7 @@ class CfgParallelRouter:self.replica = self.replica_factory()self.probed_this_render = Trueself.pairs = list(zip(module.modules(), self.replica.modules()))+ self.shared_states = _shared_states(self.pairs)self.worker = _ReplicaWorker(self.secondary)weakref.finalize(self, self.worker.close)self.stats["replica_build_s"] = round(time.perf_counter() - t0, 3)@@ -3429,7 +3459,7 @@ class CfgParallelRouter:# since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).self.prefetched.clear()_mirror_hook_state(self.pairs)- sig = _call_signature(args, kwargs)+ sig = self._signature(args, kwargs)r_args = self._to_replica(args)r_kwargs = self._to_replica(kwargs)replica = self.replicadiff --git a/studio/backend/core/inference/diffusion_nvfp4_protect.py b/studio/backend/core/inference/diffusion_nvfp4_protect.pyindex b48a58be206..ba0bf8f977e 100644--- a/studio/backend/core/inference/diffusion_nvfp4_protect.py+++ b/studio/backend/core/inference/diffusion_nvfp4_protect.py@@ -173,6 +173,10 @@ class NVFP4StepController:)return self.steps+ def compile_key(self) -> tuple:+ """What a compiled ``NVFP4Linear.forward`` guards on (``armed and protected``): a change recompiles."""+ return (bool(self.armed), bool(self.armed and self.protected))+def advance(self) -> int:self.index += 1self.protected = self.index in self.stepsdiff --git a/studio/backend/tests/test_diffusion_cfg_parallel_tier.py b/studio/backend/tests/test_diffusion_cfg_parallel_tier.pyindex 9412adea7e2..aad40d2ab63 100644--- a/studio/backend/tests/test_diffusion_cfg_parallel_tier.py+++ b/studio/backend/tests/test_diffusion_cfg_parallel_tier.py@@ -922,3 +922,91 @@ def test_a_prediction_new_to_the_replica_is_not_launched_beside_the_primary_branassert torch.equal(out, reference)assert _FX[0].errors == [] and router.disabled_reason is Noneassert router.stats["replica_calls"] == 4 and router.stats["predicted_hits"] == 3+++class _ProtectedDit(_TracedDit):+ """A compiled forward that guards on the NVFP4 protect schedule: a step that flips it recompiles."""++ def __init__(self, ctl):+ super().__init__()+ self.protect = ctl++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ side = (+ "replica"+ if threading.current_thread().name.startswith("unsloth-cfg-replica")+ else "primary"+ )+ protected = bool(self.protect.armed and self.protect.protected)+ key = (side, tuple(hidden_states.shape), tuple(encoder_hidden_states.shape), protected)+ _FX[0].enter(side, key)+ return _Dit.forward(self, hidden_states, timestep, encoder_hidden_states, return_dict)+++def _protect_controller():+ from core.inference.diffusion_nvfp4_protect import NVFP4StepController++ ctl = NVFP4StepController("2")+ ctl.begin(4)+ return ctl+++def test_a_pair_whose_protect_state_flips_never_overlaps_its_recompile():+ torch.manual_seed(0)+ ctl = _protect_controller()+ dit = _ProtectedDit(ctl)+ x, cond, uncond = torch.randn(2, 4), torch.randn(3, 4), torch.randn(5, 4)+ _FX[0] = _FxTrace(waiter = "primary")+ router, replicas = _router(dit)+ with torch.no_grad():+ for i in range(4):+ t = torch.tensor(1.0 - i / 4)+ kw = dict(hidden_states = x, timestep = t, encoder_hidden_states = uncond)+ pair = router.run_pair(+ lambda: dit(hidden_states = x, timestep = t, encoder_hidden_states = cond), (), kw+ )+ assert pair is not None and pair[1] is not None, router.disabled_reason+ ctl.advance()+ assert replicas and replicas[0].protect is ctl+ # step 2 is protected: both sides compile that state, one after the other+ assert ("primary", (2, 4), (3, 4), True) in _FX[0].compiled+ assert ("replica", (2, 4), (5, 4), True) in _FX[0].compiled+ assert _FX[0].errors == [] and router.disabled_reason is None+ assert router.stats["pairs_parallel"] == 4+++def test_a_prediction_is_not_launched_into_a_protect_state_flip():+ torch.manual_seed(0)+ ctl = _protect_controller()+ dit = _ProtectedDit(ctl)+ latents = torch.randn(2, 4)+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))++ def loop(lat):+ ctl.begin(4)+ with torch.no_grad():+ for i in range(4):+ t = torch.tensor(1.0 - i / 4)+ a = dit(hidden_states = lat, timestep = t, encoder_hidden_states = cond)[0]+ b = dit(hidden_states = lat, timestep = t, encoder_hidden_states = uncond)[0]+ lat = lat - 0.1 * (b + 4.0 * (a - b))+ ctl.advance()+ return lat++ _FX[0] = _FxTrace(compiled = {("primary", (2, 4), (3, 4), False)})+ reference = loop(latents.clone())+ _FX[0].compiled = {("primary", (2, 4), (3, 4), False)}+ _FX[0].waiter = "primary"+ router, _replicas = _router(dit)+ out = loop(latents.clone())+ assert torch.equal(out, reference)+ assert _FX[0].errors == [] and router.disabled_reason is None+ # step 0 builds the replica; steps 1-3 run their odd branch there, step 3 from a prediction+ assert router.stats["replica_calls"] == 3 and router.stats["predicted_hits"] == 1