Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio CFG-parallel: treat an NVFP4 precision-state flip as a cold call so the two branches never compile at once (#14854)

Daniel Han6d2e6bf92aed
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index c5d1041602d..e0371131d5c 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -2017,9 +2017,8 @@ def _copy_module(module: Any, device: Any) -> Any:
memo[id(hook)] = _DROP
# Per-step precision state (the NVFP4 W4A16 protect schedule) is advanced by the source's scheduler: the copy
# must read the same controller, or the two CFG branches of one step run different GEMMs.
- for value in sub.__dict__.values():
- if type(value).__name__ in _SHARED_STATE_TYPES:
- memo[id(value)] = value
+ for value in _shared_state_values(sub):
+ memo[id(value)] = value
guard = module.__dict__.get("_unsloth_compile_guard")
if guard is not None:
memo[id(guard)] = _DROP
@@ -2094,6 +2093,10 @@ _SHARED_STATE_TYPES = frozenset({"NVFP4StepController"})
_SOURCE_ONLY_ATTRS = frozenset({"_unsloth_cond_cache", "_unsloth_rocm_flash_replicas"})
+def _shared_state_values(sub: Any) -> list:
+ return [value for value in sub.__dict__.values() if type(value).__name__ in _SHARED_STATE_TYPES]
+
+
def _rewrap_compiled(source: Any, replica: Any) -> None:
"""Re-apply the wrappers Studio's compile path puts around the source's compiled blocks (``dst.compile`` replaced
them): the compile guard (eager fallback on a compile-time failure, the background-compile eager gate, the compile
@@ -2679,6 +2682,25 @@ def _call_signature(args: tuple, kwargs: dict) -> tuple:
)
+def _shared_states(pairs: list) -> tuple:
+ found: dict = {}
+ for pair in pairs:
+ for sub in pair:
+ for value in _shared_state_values(sub):
+ found.setdefault(id(value), value)
+ return tuple(found.values())
+
+
+def _shared_state_key(states: tuple) -> tuple:
+ key = []
+ for state in states:
+ try:
+ key.append(state.compile_key())
+ except Exception: # noqa: BLE001 - an unreadable state is never warm
+ key.append(object())
+ return tuple(key)
+
+
def _note_warm(seen: set, sig: tuple) -> None:
if len(seen) >= 256:
seen.clear()
@@ -2779,6 +2801,8 @@ class CfgParallelRouter:
self.probe_index: dict = {}
self.probed_this_render = False
self.pairs: list = []
+ # Shared per-step states' compile guards join every warm-call signature.
+ self.shared_states: tuple = ()
# A call new to either side may compile (FX tracing is process-wide up to torch 2.13): it never overlaps the other side.
self.warm_primary: set = set()
self.warm_replica: set = set()
@@ -2822,6 +2846,7 @@ class CfgParallelRouter:
self.kv_mirrors.clear()
self.kv_declined.clear()
self.pairs = []
+ self.shared_states = ()
self.warm_primary.clear()
self.warm_replica.clear()
self.prefetched.clear()
@@ -2865,12 +2890,16 @@ class CfgParallelRouter:
self.exclusion_refused = False
return False
+ def _signature(self, args: tuple, kwargs: dict) -> tuple:
+ """``_call_signature`` plus guarded model state: an NVFP4 protect flip recompiles under seen args."""
+ return (*_call_signature(args, kwargs), _shared_state_key(self.shared_states))
+
def _primary(self, *args: Any, **kwargs: Any) -> Any:
inner = self.inner
return (inner if inner is not None else self.orig_forward)(*args, **kwargs)
def _primary_noting(self, sig: Optional[tuple], args: tuple, kwargs: dict) -> Any:
- sig = _call_signature(args, kwargs) if sig is None else sig
+ sig = self._signature(args, kwargs) if sig is None else sig
job = self._pair_job
if sig not in self.warm_primary and job is not None:
job["event"].wait() # may compile: not while the replica runs
@@ -2909,7 +2938,7 @@ class CfgParallelRouter:
# Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after
# it would wait for it on the source stream and serialise the two branches.
self._prefetch(args, kwargs)
- sig = _call_signature(args, kwargs)
+ sig = self._signature(args, kwargs)
self._launch_prediction(latent, sig)
if (
self.branch == 0
@@ -3060,7 +3089,7 @@ class CfgParallelRouter:
return None
self.prefetched.clear()
hook_state = _mirror_hook_state(self.pairs)
- sig = _call_signature(args, kwargs)
+ sig = self._signature(args, kwargs)
try:
r_args = self._to_replica(args)
r_kwargs = self._to_replica(kwargs)
@@ -3154,7 +3183,7 @@ class CfgParallelRouter:
return
if (
sig not in self.warm_primary
- or _call_signature(p_args, p_kwargs) not in self.warm_replica
+ or self._signature(p_args, p_kwargs) not in self.warm_replica
):
return # a side may compile: the odd branch runs unpredicted, after the primary one
try:
@@ -3304,6 +3333,7 @@ class CfgParallelRouter:
self.replica = self.replica_factory()
self.probed_this_render = True
self.pairs = list(zip(module.modules(), self.replica.modules()))
+ self.shared_states = _shared_states(self.pairs)
self.worker = _ReplicaWorker(self.secondary)
weakref.finalize(self, self.worker.close)
self.stats["replica_build_s"] = round(time.perf_counter() - t0, 3)
@@ -3429,7 +3459,7 @@ class CfgParallelRouter:
# since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).
self.prefetched.clear()
_mirror_hook_state(self.pairs)
- sig = _call_signature(args, kwargs)
+ sig = self._signature(args, kwargs)
r_args = self._to_replica(args)
r_kwargs = self._to_replica(kwargs)
replica = self.replica
diff --git a/studio/backend/core/inference/diffusion_nvfp4_protect.py b/studio/backend/core/inference/diffusion_nvfp4_protect.py
index b48a58be206..ba0bf8f977e 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_protect.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_protect.py
@@ -173,6 +173,10 @@ class NVFP4StepController:
)
return self.steps
+ def compile_key(self) -> tuple:
+ """What a compiled ``NVFP4Linear.forward`` guards on (``armed and protected``): a change recompiles."""
+ return (bool(self.armed), bool(self.armed and self.protected))
+
def advance(self) -> int:
self.index += 1
self.protected = self.index in self.steps
diff --git a/studio/backend/tests/test_diffusion_cfg_parallel_tier.py b/studio/backend/tests/test_diffusion_cfg_parallel_tier.py
index 9412adea7e2..aad40d2ab63 100644
--- a/studio/backend/tests/test_diffusion_cfg_parallel_tier.py
+++ b/studio/backend/tests/test_diffusion_cfg_parallel_tier.py
@@ -922,3 +922,91 @@ def test_a_prediction_new_to_the_replica_is_not_launched_beside_the_primary_bran
assert torch.equal(out, reference)
assert _FX[0].errors == [] and router.disabled_reason is None
assert router.stats["replica_calls"] == 4 and router.stats["predicted_hits"] == 3
+
+
+class _ProtectedDit(_TracedDit):
+ """A compiled forward that guards on the NVFP4 protect schedule: a step that flips it recompiles."""
+
+ def __init__(self, ctl):
+ super().__init__()
+ self.protect = ctl
+
+ def forward(
+ self,
+ hidden_states,
+ timestep,
+ encoder_hidden_states,
+ return_dict = False,
+ ):
+ side = (
+ "replica"
+ if threading.current_thread().name.startswith("unsloth-cfg-replica")
+ else "primary"
+ )
+ protected = bool(self.protect.armed and self.protect.protected)
+ key = (side, tuple(hidden_states.shape), tuple(encoder_hidden_states.shape), protected)
+ _FX[0].enter(side, key)
+ return _Dit.forward(self, hidden_states, timestep, encoder_hidden_states, return_dict)
+
+
+def _protect_controller():
+ from core.inference.diffusion_nvfp4_protect import NVFP4StepController
+
+ ctl = NVFP4StepController("2")
+ ctl.begin(4)
+ return ctl
+
+
+def test_a_pair_whose_protect_state_flips_never_overlaps_its_recompile():
+ torch.manual_seed(0)
+ ctl = _protect_controller()
+ dit = _ProtectedDit(ctl)
+ x, cond, uncond = torch.randn(2, 4), torch.randn(3, 4), torch.randn(5, 4)
+ _FX[0] = _FxTrace(waiter = "primary")
+ router, replicas = _router(dit)
+ with torch.no_grad():
+ for i in range(4):
+ t = torch.tensor(1.0 - i / 4)
+ kw = dict(hidden_states = x, timestep = t, encoder_hidden_states = uncond)
+ pair = router.run_pair(
+ lambda: dit(hidden_states = x, timestep = t, encoder_hidden_states = cond), (), kw
+ )
+ assert pair is not None and pair[1] is not None, router.disabled_reason
+ ctl.advance()
+ assert replicas and replicas[0].protect is ctl
+ # step 2 is protected: both sides compile that state, one after the other
+ assert ("primary", (2, 4), (3, 4), True) in _FX[0].compiled
+ assert ("replica", (2, 4), (5, 4), True) in _FX[0].compiled
+ assert _FX[0].errors == [] and router.disabled_reason is None
+ assert router.stats["pairs_parallel"] == 4
+
+
+def test_a_prediction_is_not_launched_into_a_protect_state_flip():
+ torch.manual_seed(0)
+ ctl = _protect_controller()
+ dit = _ProtectedDit(ctl)
+ latents = torch.randn(2, 4)
+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))
+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))
+
+ def loop(lat):
+ ctl.begin(4)
+ with torch.no_grad():
+ for i in range(4):
+ t = torch.tensor(1.0 - i / 4)
+ a = dit(hidden_states = lat, timestep = t, encoder_hidden_states = cond)[0]
+ b = dit(hidden_states = lat, timestep = t, encoder_hidden_states = uncond)[0]
+ lat = lat - 0.1 * (b + 4.0 * (a - b))
+ ctl.advance()
+ return lat
+
+ _FX[0] = _FxTrace(compiled = {("primary", (2, 4), (3, 4), False)})
+ reference = loop(latents.clone())
+ _FX[0].compiled = {("primary", (2, 4), (3, 4), False)}
+ _FX[0].waiter = "primary"
+ router, _replicas = _router(dit)
+ out = loop(latents.clone())
+ assert torch.equal(out, reference)
+ assert _FX[0].errors == [] and router.disabled_reason is None
+ # step 0 builds the replica; steps 1-3 run their odd branch there, step 3 from a prediction
+ assert router.stats["replica_calls"] == 3 and router.stats["predicted_hits"] == 1