Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep CFG-parallel on the default speed tier and run Z-Image's guidance branches on two cards (#14564)
Daniel Han543386302f6e
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex 638d6f849a3..80302c9bd08 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -575,7 +575,12 @@ class BlockGraph:return outdef _record(self, full: tuple, live: list) -> Optional[_Entry]:- from .diffusion_cuda_graph import _capturing, refuse_if_exhausted, retire_failed_capture+ from .diffusion_cuda_graph import (+ CaptureRefused,+ _capturing,+ refuse_if_exhausted,+ retire_failed_capture,+ )torch = _torch()key = full[0]@@ -626,6 +631,11 @@ class BlockGraph:del out, flatcurrent.wait_stream(stream)self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)+ except CaptureRefused:+ # Refused before capture_begin: nothing recorded, the pool untouched. Eager now, record on a later call.+ self.seen[full] = (out_spec, metas)+ self.stats["capture_refusals"] = self.stats.get("capture_refusals", 0) + 1+ return Noneexcept Exception as exc: # noqa: BLE001 - a block that cannot record runs its compute for the load's lifeself._poison(exc)return None@@ -1013,6 +1023,7 @@ class SegmentGraph:def _capture(self, full: tuple, live: list) -> Optional[_SegEntry]:from .diffusion_cuda_graph import (+ CaptureRefused,_capturing,capture_device,note_capture,@@ -1053,6 +1064,10 @@ class SegmentGraph:raisecurrent.wait_stream(stream)self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)+ except CaptureRefused:+ self.seen[full] = rec+ self.stats["segment_refusals"] = self.stats.get("segment_refusals", 0) + 1+ return Noneexcept Exception as exc: # noqa: BLE001 - this layout stays per block for the load's lifeself.capture_error = {"type": type(exc).__name__, "msg": str(exc)[:4000]}self.refused.add(full)diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 8148e4ecb7e..b74828a898b 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -138,6 +138,73 @@ def _disabled_reason() -> str:# and neither holds the lock while it records, so captures do not serialise against each other._CAPTURE_LOCK = threading.Lock()_CAPTURE_DEPTH = 0+# CFG-parallel replica work is counted from queueing until its inputs drop; a capture waits for the count to drain.+_CAPTURE_IDLE = threading.Condition(_CAPTURE_LOCK)+_OUTSIDE_JOBS = 0+_OUTSIDE_LOCAL = threading.local()+# A count that never drains (dead worker) must not hang captures: after this long the capture is refused and the step+# runs eager. Symmetrically, replica work that waits this long on a recording is refused and its branch runs on the source.+OUTSIDE_WAIT_S = 120.0+# Set by a refusal, cleared once the replica-side count drains: while it stands, captures refuse without waiting again.+_OUTSIDE_STUCK = [False]+# Holds owned by threads parked in ``_capturing`` waiting to record: a parked thread issues no CUDA calls, so another+# held thread may record while it waits (two replica workers that both want to capture do not wait each other out).+_OUTSIDE_PARKED = [0]+++class CaptureExclusionTimeout(RuntimeError):+ """A capture or a replica-side job gave up waiting on the other side of the capture exclusion."""+++class CaptureRefused(CaptureExclusionTimeout):+ """Raised by ``_capturing`` BEFORE the depth is raised and before any recording begins: nothing to clean up."""+++class OutsideWorkRefused(CaptureExclusionTimeout):+ """Raised by ``held_work`` before the replica-side work runs: none of its CUDA calls were made."""+++def _other_holds(own: int, parked: int) -> int:+ """Replica-side holds a capture by this thread must wait for: its own held work (``own``) is discounted, and a+ held thread also discounts holds whose threads are parked waiting to record (``parked``, this thread's excluded)."""+ return _OUTSIDE_JOBS - own - (parked if own else 0)+++def capture_refused_now() -> bool:+ """Whether a capture would be refused without waiting (a stale replica-side count is still standing)."""+ with _CAPTURE_IDLE:+ own = getattr(_OUTSIDE_LOCAL, "depth", 0)+ return _OUTSIDE_STUCK[0] and _other_holds(own, _OUTSIDE_PARKED[0]) > 0+++def _wait_outside_drained(own: int) -> None:+ """Under ``_CAPTURE_IDLE``: wait for other threads' held work to drain (a global capture forbids its CUDA calls), else raise ``CaptureRefused``."""+ drained = _other_holds(own, _OUTSIDE_PARKED[0]) <= 0+ if not drained and not _OUTSIDE_STUCK[0]:+ if own:+ _OUTSIDE_PARKED[0] += own+ _CAPTURE_IDLE.notify_all()+ try:+ drained = _CAPTURE_IDLE.wait_for(+ lambda: _other_holds(own, _OUTSIDE_PARKED[0] - own) <= 0,+ timeout = OUTSIDE_WAIT_S,+ )+ finally:+ if own:+ _OUTSIDE_PARKED[0] -= own+ _CAPTURE_IDLE.notify_all()+ if not drained:+ _OUTSIDE_STUCK[0] = True+ raise CaptureRefused(+ f"replica-side CUDA work still running after {OUTSIDE_WAIT_S:g}s; capture refused"+ )+++def await_outside_drained() -> None:+ """Wait as a capture does, for the warm-ups: they may compile, and up to torch 2.13 FX tracing is flagged process-wide."""+ own = getattr(_OUTSIDE_LOCAL, "depth", 0)+ with _CAPTURE_IDLE:+ _wait_outside_drained(own)@contextlib.contextmanager@@ -145,16 +212,62 @@ def _capturing():"""Mark a capture as recording for the duration of the block.The depth MUST be raised before ``torch.cuda.graph`` is entered, which is what makes the- ordering above hold; keep this the outermost of the two context managers.+ ordering above hold; keep this the outermost of the two context managers. Raises+ ``CaptureRefused`` (depth untouched, nothing recorded) when replica-side work did not drain in time."""global _CAPTURE_DEPTH- with _CAPTURE_LOCK:+ own = getattr(_OUTSIDE_LOCAL, "depth", 0)+ with _CAPTURE_IDLE:+ _wait_outside_drained(own)_CAPTURE_DEPTH += 1try:yieldfinally:- with _CAPTURE_LOCK:+ with _CAPTURE_IDLE:_CAPTURE_DEPTH -= 1+ _CAPTURE_IDLE.notify_all()+++def capture_hold() -> None:+ global _OUTSIDE_JOBS+ with _CAPTURE_IDLE:+ _OUTSIDE_JOBS += 1+++def capture_release() -> None:+ global _OUTSIDE_JOBS+ with _CAPTURE_IDLE:+ _OUTSIDE_JOBS = max(0, _OUTSIDE_JOBS - 1)+ if _OUTSIDE_JOBS == 0:+ _OUTSIDE_STUCK[0] = False+ _CAPTURE_IDLE.notify_all()+++def held_work():+ """Waits for another thread's recording; this thread may then record without waiting on its own hold.++ Raises ``OutsideWorkRefused`` (the body never runs) when a recording is still going after ``OUTSIDE_WAIT_S``."""+ with _CAPTURE_IDLE:+ if not _CAPTURE_IDLE.wait_for(lambda: _CAPTURE_DEPTH == 0, timeout = OUTSIDE_WAIT_S):+ raise OutsideWorkRefused(+ f"a CUDA graph capture still recording after {OUTSIDE_WAIT_S:g}s; replica work refused"+ )+ _OUTSIDE_LOCAL.depth = getattr(_OUTSIDE_LOCAL, "depth", 0) + 1+ try:+ yield+ finally:+ _OUTSIDE_LOCAL.depth -= 1+++def outside_capture():+ capture_hold()+ try:+ with held_work():+ yield+ finally:+ capture_release()@contextlib.contextmanager@@ -641,6 +754,17 @@ def _outer_layer(module: Any) -> Any:return slot if getattr(slot, "_unsloth_outer_forward", False) is True else None+def _router_layer(module: Any) -> Any:+ try:+ slot = module.__dict__.get("forward")+ except Exception: # noqa: BLE001+ return None+ # A static step skip installed after the router wraps it: the router is the skip's inner.+ if getattr(slot, "_unsloth_outer_forward", False) is True:+ slot = getattr(slot, "inner", None)+ return slot if getattr(type(slot), "_unsloth_cfg_router_layer", False) is True else None++class _Entry:"""One captured graph plus the static buffers it replays into."""@@ -800,10 +924,14 @@ class GraphedForward:return bool(getattr(self, "_sealed_scopes", None))def install(self) -> "GraphedForward":- """Set ``forward`` via ``__dict__`` (``__setattr__`` would inspect it), under any outer step-skip layer."""+ """Set ``forward`` via ``__dict__`` (``__setattr__`` would inspect it), under any step-skip layer or CFG router."""if self.placement is not None:self.placement.install(self)return self+ router = _router_layer(self.module)+ if router is not None:+ router.inner = self+ return selfouter = _outer_layer(self.module)if outer is not None:outer.inner = self@@ -815,6 +943,10 @@ class GraphedForward:if self.placement is not None:self.placement.uninstall(self)return self+ router = _router_layer(self.module)+ if router is not None and router.inner is self:+ router.inner = None+ return selfouter = _outer_layer(self.module)if outer is not None:if outer.inner is self:@@ -1274,6 +1406,16 @@ class GraphedForward:stream_before = _current_stream()try:captured = self._capture(args, kwargs, call, key)+ except CaptureRefused as exc:+ _restore_stream(stream_before)+ self.stats["capture_refusals"] = self.stats.get("capture_refusals", 0) + 1+ if self.logger is not None:+ self.logger.warning(+ "diffusion.cuda_graph: capture deferred on %s (%s); this step runs eager",+ type(self.module).__name__,+ exc,+ )+ exc.__traceback__ = Noneexcept Exception as exc: # noqa: BLE001 - a failed capture must never fail the render# torch.cuda.graph's __exit__ raises in capture_end before it leaves its capture stream, so the thread# would run everything after (the eager retry, the rest of the process) on the capture stream.@@ -1443,6 +1585,9 @@ class GraphedForward:refuse_if_exhausted(self.logger) # before the warm-ups, and before the step skip's counters are saved+ if capture_refused_now():+ raise CaptureRefused("replica-side CUDA work still running; capture refused")+ await_outside_drained()# Under an offload placement the capture also records the copy-stream onloads (OffloadPlacement.record).record = call if self.placement is None else self.placement.recorder(call)# Side stream: a workspace first created DURING capture is only valid while recording.@@ -1494,7 +1639,9 @@ class GraphedForward:with _capturing(), recorder(graph, pool = pool):out = record(*static_args, **static_kwargs)except BaseException as exc:- retire_failed_capture(graph, pool, exc)+ # A refusal is raised before the recording begins: the graph and the pool were never touched.+ if not isinstance(exc, CaptureRefused):+ retire_failed_capture(graph, pool, exc)raisefinally:if skip_stats is not None:diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex 418f92ee509..d5bb17d208b 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -1217,6 +1217,9 @@ def _hooks_present(module: Any) -> Optional[str]:override = sub.__dict__.get("forward")if override is None or isinstance(override, (types.MethodType, CfgParallelRouter)):continue+ if not name and _resident_graph(sub) is not None:+ # install_cfg_parallel moves the root's graph under the router+ continueif kind == "offload" or _is_offload_object(override):continueif callable(override) and not _trusted_hook(override):@@ -1462,6 +1465,13 @@ def _iter_tensors(obj: Any, depth: int = 0) -> Iterator[Any]:yield from _iter_tensors(v, depth + 1)+def _to_primary(out: Any, primary: Any) -> Any:+ """Copy a replica result home with captures held off: the job's own hold dropped when the worker finished."""+ from .diffusion_cuda_graph import outside_capture+ with outside_capture():+ return move_to_device(out, primary)++class _ReplicaWorker:"""One daemon thread bound to the replica's card: set_device, grad / inference / autocast mode and the caller'scontext variables (inductor overrides live in one on torch >= 2.12) are applied per job."""@@ -1472,25 +1482,50 @@ class _ReplicaWorker:self._thread = threading.Thread(target = self._loop, name = f"unsloth-cfg-replica-{device}", daemon = True)- self._thread.start()+ # Global capture mode forbids this thread's CUDA allocations while any recording runs, so hold them out.+ from .diffusion_cuda_graph import capture_hold, capture_release++ capture_hold()+ try:+ self._thread.start()+ except BaseException:+ capture_release()+ raisedef _loop(self) -> None:+ from .diffusion_cuda_graph import capture_release, held_work+torch = _torch()- if getattr(self.device, "type", None) == "cuda":- try:- torch.cuda.set_device(self.device)- except Exception: # noqa: BLE001- pass+ try:+ with held_work():+ if getattr(self.device, "type", None) == "cuda":+ try:+ torch.cuda.set_device(self.device)+ except Exception: # noqa: BLE001+ pass+ except Exception: # noqa: BLE001 - refused while a capture records: each job binds its device anyway+ pass+ finally:+ capture_release()while True:job = self._jobs.get()if job is None:returnfn, ctx, modes, done = job+ job = Nonetry:- done["result"] = ctx.run(self._run, fn, modes)- except BaseException as exc: # noqa: BLE001 - surfaced on the caller's thread- done["error"] = exc+ try:+ with held_work():+ try:+ done["result"] = ctx.run(self._run, fn, modes)+ except BaseException as exc: # noqa: BLE001 - surfaced on the caller's thread+ done["error"] = exc+ fn = ctx = None+ except Exception as exc: # noqa: BLE001 - refused while a capture records: the job never ran+ done["error"] = exc+ fn = ctx = Nonefinally:+ capture_release()done["event"].set()def _run(self, fn: Callable[[], Any], modes: dict) -> Any:@@ -1519,6 +1554,9 @@ class _ReplicaWorker:"autocast": autocast,}done: dict[str, Any] = {"event": threading.Event()}+ from .diffusion_cuda_graph import capture_hold++ capture_hold()self._jobs.put((fn, contextvars.copy_context(), modes, done))return done@@ -1679,6 +1717,39 @@ def _mirror_hook_state(pairs: list) -> tuple:return state+def _call_signature(args: tuple, kwargs: dict) -> tuple:+ """What a compiled forward's guards mostly key on: tensor layout, flags and modes (never per-step scalars)."""+ torch = _torch()+ sig: list = []+ for path, leaf in _leaves((args, kwargs)):+ if isinstance(leaf, torch.Tensor):+ try:+ stride = tuple(leaf.stride())+ except Exception: # noqa: BLE001 - layouts without strides+ stride = None+ sig.append(+ (+ path,+ tuple(leaf.shape),+ stride,+ leaf.dtype,+ leaf.requires_grad,+ leaf.is_inference(),+ )+ )+ elif leaf is None or isinstance(leaf, (bool, str)):+ sig.append((path, leaf))+ else:+ sig.append((path, type(leaf)))+ return tuple(sig), torch.is_inference_mode_enabled(), torch.is_grad_enabled()+++def _note_warm(seen: set, sig: tuple) -> None:+ if len(seen) >= 256:+ seen.clear()+ seen.add(sig)++def _background_compile_call() -> bool:try:from .diffusion_bg_compile import capture_suppressed, eager_forced@@ -1695,7 +1766,11 @@ class CfgParallelRouter:step's; the rest -- negative embeds, masks -- kept) and launched on the replica BEFORE the primary branch runs, sothe two cards overlap even when the primary forward blocks the host. The real call must then pass exactly thepredicted tensors (identity) and equal plain values, else the prediction is discarded and the branch runs- unpredicted."""+ unpredicted.++ A whole-step CUDA graph installs under the router (``inner``); split callers (Z-Image) use :meth:`run_pair`."""++ _unsloth_cfg_router_layer = Truedef __init__(self,@@ -1709,6 +1784,8 @@ class CfgParallelRouter:torch = _torch()self.module_ref = weakref.ref(module)self.orig_forward = module.forward+ self.inner: Any = None+ self._pairing = Falseself.had_override = "forward" in module.__dict__self.primary = torch.device(primary)self.secondary = torch.device(secondary)@@ -1735,12 +1812,21 @@ class CfgParallelRouter:"predicted_hits": 0,"predicted_misses": 0,"opaque_primary_calls": 0,+ "pairs_parallel": 0,+ "exclusion_skips": 0,}+ # Set when replica work was refused by a long recording: until that capture ends, later replica attempts go+ # straight to the source instead of each waiting OUTSIDE_WAIT_S again.+ self.exclusion_refused = Falseself.template: Optional[tuple[Any, Any, dict]] = Noneself.pending: Optional[dict] = Noneself.b0_leaves: dict = {}self.fingerprint: Optional[int] = Noneself.pairs: list = []+ # A call new to either side may compile (FX tracing is process-wide up to torch 2.13): it never overlaps the other side.+ self.warm_primary: set = set()+ self.warm_replica: set = set()+ self._pair_job: Optional[dict] = Nonedef describe(self) -> dict[str, Any]:return {@@ -1767,6 +1853,8 @@ class CfgParallelRouter:self._drop_pending()self.replica = Noneself.pairs = []+ self.warm_primary.clear()+ self.warm_replica.clear()self.prefetched.clear()self.carry = []self.next_carry = []@@ -1787,14 +1875,48 @@ class CfgParallelRouter:self.cfg_seen = Falseself.singles = 0+ def _note_refusal(self, exc: BaseException) -> None:+ from .diffusion_cuda_graph import OutsideWorkRefused+ if isinstance(exc, OutsideWorkRefused):+ self.exclusion_refused = True++ def _refused_this_capture(self) -> bool:+ """Whether replica work was refused and the capture that refused it is still recording."""+ if not self.exclusion_refused:+ return False+ from .diffusion_cuda_graph import capture_in_progress++ if capture_in_progress():+ self.stats["exclusion_skips"] += 1+ return True+ self.exclusion_refused = False+ return False++ def _primary(self, *args: Any, **kwargs: Any) -> Any:+ inner = self.inner+ return (inner if inner is not None else self.orig_forward)(*args, **kwargs)++ def _primary_noting(self, sig: Optional[tuple], args: tuple, kwargs: dict) -> Any:+ sig = _call_signature(args, kwargs) if sig is None else sig+ job = self._pair_job+ if sig not in self.warm_primary and job is not None:+ job["event"].wait() # may compile: not while the replica runs+ out = self._primary(*args, **kwargs)+ _note_warm(self.warm_primary, sig)+ return out+def __call__(self, *args: Any, **kwargs: Any) -> Any:+ if self._pairing:+ self.stats["primary_calls"] += 1+ return self._primary_noting(None, args, kwargs)if self.disabled_reason is not None or _background_compile_call():# Disabled: no branch state to keep. A render that must run eager while its compile is in flight, or the# background compile thread itself replaying recorded inputs: both branches stay on the source, which# keeps that render bit-identical to an eager one and leaves the router's per-render state untouched.self.stats["primary_calls"] += 1- return self.orig_forward(*args, **kwargs)+ return self._primary(*args, **kwargs)latent = _latent_arg(args, kwargs)+ sig = Noneif latent is not None and self.prev_latent is not None and latent is self.prev_latent:self.branch += 1else:@@ -1814,7 +1936,8 @@ class CfgParallelRouter:# Copy this step's inputs to the replica card BEFORE the primary branch is queued: a copy queued after# it would wait for it on the source stream and serialise the two branches.self._prefetch(args, kwargs)- self._launch_prediction(latent)+ sig = _call_signature(args, kwargs)+ self._launch_prediction(latent, sig)if (self.branch % 2 == 1and self.disabled_reason is None@@ -1830,6 +1953,9 @@ class CfgParallelRouter:self._build_replica()if self.branch == 1:self._record_template(args, kwargs, latent)+ elif self.cfg_seen and self._refused_this_capture():+ # this step's prefetch or prediction was refused: the branch runs on the source, no further wait+ self._drop_pending()elif self.cfg_seen:if self.branch == 1:self._record_template(args, kwargs, latent)@@ -1842,10 +1968,77 @@ class CfgParallelRouter:self.stats["steps_parallel"] += 1return outexcept Exception as exc: # noqa: BLE001 - this branch is recomputed on the primary below+ self._note_refusal(exc)self.disable(f"replica call failed: {type(exc).__name__}: {exc}")self.cfg_seen = Trueself.stats["primary_calls"] += 1- return self.orig_forward(*args, **kwargs)+ return self._primary_noting(sig, args, kwargs)++ def run_pair(+ self, primary_call: Callable[[], Any], args: tuple, kwargs: dict+ ) -> Optional[tuple]:+ """Second half on the replica, ``primary_call`` on the source. ``(first, second)``; None = run both on source;+ ``second`` None = replica failed."""+ if self.disabled_reason is not None or _background_compile_call():+ return None+ module = self.module_ref()+ if module is None or _holds_opaque_tensors((args, kwargs)):+ return None+ if any(getattr(module, attr, None) for attr in _ROOT_HOOK_DICTS):+ return None+ self._drop_pending()+ self.template = None+ self.prev_latent = None+ self.branch = 0+ if self.replica is not None:+ self._drop_if_stale()+ if self.replica is None and self.disabled_reason is None:+ self._build_replica()+ if self.replica is None or self.worker is None or self.disabled_reason is not None:+ return None+ if self._refused_this_capture():+ return None+ self.prefetched.clear()+ hook_state = _mirror_hook_state(self.pairs)+ sig = _call_signature(args, kwargs)+ try:+ r_args = self._to_replica(args)+ r_kwargs = self._to_replica(kwargs)+ except Exception as exc: # noqa: BLE001 - e.g. no room left on the replica card: both halves run on the source+ self.disable(f"replica input copy failed: {type(exc).__name__}: {exc}")+ return None+ finally:+ self.next_carry = []+ self.prefetched.clear()+ job = self.worker.submit(_replica_call(self.replica, r_args, r_kwargs))+ if sig not in self.warm_replica:+ job["event"].wait() # may compile: the primary half waits+ self._pairing = True+ self._pair_job = job+ try:+ first = primary_call()+ except BaseException:+ with contextlib.suppress(BaseException):+ _ReplicaWorker.wait(job)+ raise+ finally:+ self._pairing = False+ self._pair_job = None+ try:+ second = _ReplicaWorker.wait(job)+ if _hook_state(self.pairs) != hook_state:+ # source hook state moved during the first half: rerun+ _mirror_hook_state(self.pairs)+ second = self.worker.call(_replica_call(self.replica, r_args, r_kwargs))+ second = _to_primary(second, self.primary)+ except Exception as exc: # noqa: BLE001 - the caller recomputes the second half on the source+ self.disable(f"replica call failed: {type(exc).__name__}: {exc}")+ return first, None+ _note_warm(self.warm_replica, sig)+ self.stats["replica_calls"] += 1+ self.stats["steps_parallel"] += 1+ self.stats["pairs_parallel"] += 1+ return first, seconddef _record_template(self, args: tuple, kwargs: dict, latent: Any) -> None:torch = _torch()@@ -1888,15 +2081,23 @@ class CfgParallelRouter:except KeyError:return None- def _launch_prediction(self, latent: Any) -> None:+ def _launch_prediction(self, latent: Any, sig: tuple) -> None:+ if self._refused_this_capture():+ returnpredicted = self._predict(latent)if predicted is None or self.worker is None:returnp_args, p_kwargs = predicted+ if (+ sig not in self.warm_primary+ or _call_signature(p_args, p_kwargs) not in self.warm_replica+ ):+ return # a side may compile: the odd branch runs unpredicted, after the primary onetry:r_args = self._to_replica(p_args)r_kwargs = self._to_replica(p_kwargs)- except Exception: # noqa: BLE001 - no prediction this step; the branch runs unpredicted+ except Exception as exc: # noqa: BLE001 - no prediction this step; the branch runs unpredicted+ self._note_refusal(exc)returntorch = _torch()# What the replica consumes, kept per tensor: at the real call the sources are compared bitwise against it, so@@ -1916,7 +2117,7 @@ class CfgParallelRouter:"leaves": _leaves((p_args, p_kwargs)),"by_value": {path for path, kind in kinds.items() if kind[0] == "b0eq"},"snapshots": snapshots,- "job": self.worker.submit(lambda: replica(*r_args, **r_kwargs)),+ "job": self.worker.submit(_replica_call(replica, r_args, r_kwargs)),}def _take_prediction(self, args: tuple, kwargs: dict) -> Any:@@ -1944,7 +2145,7 @@ class CfgParallelRouter:self.prefetched.clear()return Noneself.stats["predicted_hits"] += 1- return move_to_device(out, self.primary)+ return _to_primary(out, self.primary)def _drop_pending(self) -> None:pending, self.pending = self.pending, None@@ -1980,6 +2181,8 @@ class CfgParallelRouter:module = self.module_ref()if module is None:return+ from .diffusion_cuda_graph import OutsideWorkRefused, outside_capture+t0 = time.perf_counter()try:hooked = _hooks_present(module)@@ -2001,7 +2204,8 @@ class CfgParallelRouter:f"replica needs {need_mib} MiB, {free} MiB free on {self.secondary}")self.fingerprint = weights_fingerprint(module)- with _device_context(self.secondary):+ # Copying the weights to the replica card is CUDA work another pipeline's capture prohibits.+ with outside_capture(), _device_context(self.secondary):self.replica = self.replica_factory()self.pairs = list(zip(module.modules(), self.replica.modules()))self.worker = _ReplicaWorker(self.secondary)@@ -2015,15 +2219,27 @@ class CfgParallelRouter:)except Exception as exc: # noqa: BLE001self.replica = None- with contextlib.suppress(Exception):- _torch().cuda.empty_cache()+ # A refused build allocated nothing, and its capture is still recording: no allocator call may run now.+ # Any other failure frees what it left behind, itself with captures held off.+ if not isinstance(exc, OutsideWorkRefused):+ with contextlib.suppress(Exception):+ with outside_capture():+ _torch().cuda.empty_cache()self.disable(f"replica unavailable: {exc}")def _prefetch(self, args: tuple, kwargs: dict) -> None:- for tensor in (*_iter_tensors((args, kwargs)), *self.carry):- if tensor.device == self.secondary or id(tensor) in self.prefetched:- continue- self.prefetched[id(tensor)] = (tensor, tensor.to(self.secondary))+ from .diffusion_cuda_graph import outside_capture+ if self._refused_this_capture():+ return+ try:+ with outside_capture():+ for tensor in (*_iter_tensors((args, kwargs)), *self.carry):+ if tensor.device == self.secondary or id(tensor) in self.prefetched:+ continue+ self.prefetched[id(tensor)] = (tensor, tensor.to(self.secondary))+ except Exception as exc: # noqa: BLE001 - only an optimisation: no prefetch, the call copies its inputs itself+ self.prefetched.clear()+ self._note_refusal(exc)def _to_replica(self, obj: Any) -> Any:torch = _torch()@@ -2051,18 +2267,27 @@ class CfgParallelRouter:return type(value)((k, walk(v, depth + 1)) for k, v in value.items())return value- return walk(obj)+ from .diffusion_cuda_graph import outside_capture++ with outside_capture():+ return walk(obj)def _run_replica(self, args: tuple, kwargs: dict) -> Any:# Unpredicted: nothing verified the copies made before the primary branch ran, and an input edited in place# since then would reach the replica stale. Copy this call's inputs now (ordered after the primary branch).self.prefetched.clear()_mirror_hook_state(self.pairs)+ sig = _call_signature(args, kwargs)r_args = self._to_replica(args)r_kwargs = self._to_replica(kwargs)replica = self.replica- out = self.worker.call(lambda: replica(*r_args, **r_kwargs))- return move_to_device(out, self.primary)+ out = self.worker.call(_replica_call(replica, r_args, r_kwargs))+ _note_warm(self.warm_replica, sig)+ return _to_primary(out, self.primary)+++def _replica_call(replica: Any, args: tuple, kwargs: dict) -> Callable[[], Any]:+ return lambda: replica(*args, **kwargs)_CFG_ROUTERS: "weakref.WeakKeyDictionary[Any, CfgParallelRouter]" = weakref.WeakKeyDictionary()@@ -2114,6 +2339,21 @@ def cfg_parallel_decline(return None+def _resident_graph(module: Any) -> Any:+ slot = module.__dict__.get("forward")+ try:+ from .diffusion_cuda_graph import GraphedCompiledCall, GraphedForward+ except Exception: # noqa: BLE001+ return None+ if (+ type(slot) is GraphedForward+ and slot.placement is None+ and not isinstance(slot, GraphedCompiledCall)+ ):+ return slot+ return None++def install_cfg_parallel(pipe: Any,*,@@ -2157,12 +2397,18 @@ def install_cfg_parallel(if free is not None and need_mib + 1024 > free:reason = f"replica needs {need_mib} MiB, {free} MiB free on cuda:{best}"else:+ graph = _resident_graph(denoiser)+ if graph is not None:+ del denoiser.__dict__["forward"]router = CfgParallelRouter(denoiser, primary = primary, secondary = f"cuda:{best}", logger = logger)+ router.inner = graphtry:_CFG_ROUTERS[pipe] = routerexcept TypeError:+ if graph is not None:+ denoiser.__dict__["forward"] = graphreturn "pipeline cannot be tracked (not weak-referenceable)"denoiser.forward = routerdenoiser._unsloth_cfg_router = router@@ -2204,6 +2450,19 @@ def invalidate_cfg_replica(return True+def _router_slot(module: Any, router: Any) -> Any:+ """``module`` when ``router`` is its forward, the outer layer (static step skip) whose inner it is, else None."""+ slot = module.__dict__.get("forward")+ if slot is router:+ return module+ if (+ getattr(slot, "_unsloth_outer_forward", False) is True+ and getattr(slot, "inner", None) is router+ ):+ return slot+ return None++def drop_displaced_cfg_router(pipe: Any,reason: str,@@ -2218,8 +2477,12 @@ def drop_displaced_cfg_router(if router is None:return Falsemodule = router.module_ref()- if module is not None and module.__dict__.get("forward") is router:+ holder = _router_slot(module, router) if module is not None else None+ if holder is module and module is not None:return False+ if holder is not None:+ # The replica cannot carry a step skip (it holds the source module), so the router could never build one.+ reason = "a static step skip wraps the denoiser"uninstall_cfg_parallel(pipe)with contextlib.suppress(TypeError):_CFG_DECLINES[pipe] = reason@@ -2249,10 +2512,22 @@ def uninstall_cfg_parallel(pipe: Any) -> None:router.drop_replica()module = router.module_ref()if module is not None:- if module.__dict__.get("forward") is router:- if router.had_override:+ holder = _router_slot(module, router)+ if holder is module:+ if router.inner is not None:+ module.__dict__["forward"] = router.inner+ router.inner = None+ elif router.had_override:module.__dict__["forward"] = router.orig_forwardelse:del module.__dict__["forward"]+ elif holder is not None:+ # under a static step skip: hand the skip what the router wrapped+ holder.inner = (+ router.inner+ if router.inner is not None+ else (router.orig_forward if router.had_override else None)+ )+ router.inner = Noneif module.__dict__.get("_unsloth_cfg_router") is router:del module.__dict__["_unsloth_cfg_router"]diff --git a/studio/backend/core/inference/diffusion_zimage_cfg_split.py b/studio/backend/core/inference/diffusion_zimage_cfg_split.pyindex 029cd3a7fd1..bba4abeefe6 100644--- a/studio/backend/core/inference/diffusion_zimage_cfg_split.py+++ b/studio/backend/core/inference/diffusion_zimage_cfg_split.py@@ -19,7 +19,7 @@ bit-exact: a single sample with no mask can pick other attention / GEMM tiles thEngaged per render (``render``: Studio knows the images per call, so ``2 * B`` samples is a guided call, ``B`` is not),only on a resident denoiser: a block-streamed one would upload its weights twice per step, which costs more than therecompile. Declined per call under a step cache and a static step skip that cannot key a step's two calls apart.-Kill switch: ``UNSLOTH_DIFFUSION_ZIMAGE_CFG_SPLIT=0``.+Under a CFG-parallel router the halves go to ``run_pair`` (identity pairing never matches fresh tensors). Kill switch: ``UNSLOTH_DIFFUSION_ZIMAGE_CFG_SPLIT=0``."""from __future__ import annotations@@ -74,7 +74,6 @@ def _declined_now(module: Any) -> Optional[str]:for skip in (slots.get("_unsloth_static_skip"), slots.get("forward")):if getattr(skip, "_unsloth_outer_forward", False) and not _skip_keys_calls_apart(skip):return "static step skip without a per-step call order"- # No CFG-router decline: it pairs by first-tensor identity (a fresh timestep per call), so never pairs Z-Image.if getattr(module, "training", False):return "training"return None@@ -125,8 +124,19 @@ def _make_call(orig_call: Any) -> Any:stats["batched"] += 1return orig_call(self, *args, **kwargs)cond, uncond, kw = parts- out_cond = orig_call(self, *cond, **kw)- out_uncond = orig_call(self, *uncond, **kw)+ router = slots.get("_unsloth_cfg_router")+ pair = None+ if router is not None and slots.get("forward") is router:+ pair = router.run_pair(lambda: orig_call(self, *cond, **kw), uncond, dict(kw))+ if pair is not None:+ out_cond, out_uncond = pair+ if out_uncond is None:+ out_uncond = orig_call(self, *uncond, **kw)+ elif stats is not None:+ stats["parallel"] = stats.get("parallel", 0) + 1+ else:+ out_cond = orig_call(self, *cond, **kw)+ out_uncond = orig_call(self, *uncond, **kw)if stats is not None:stats["split"] += 1return _join(out_cond, out_uncond)diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.pyindex 796df8bfd79..f0b7a1bccfe 100644--- a/studio/backend/tests/test_diffusion_block_graph.py+++ b/studio/backend/tests/test_diffusion_block_graph.py@@ -1065,3 +1065,30 @@ def test_a_model_offloaded_denoiser_keeps_a_specific_whole_step_refusal(monkeypareason = pipe._unsloth_cuda_graph_reasonassert handles == () and reason.startswith("forward is not capture-safe")assert cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV not in reason and bg.BLOCK_GRAPHS_ENV in reason+++def test_a_refused_block_recording_runs_eager_and_records_later(monkeypatch):+ _cuda()+ from core.inference import diffusion_cuda_graph as cg++ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ net = _net(blocks = 2).cuda()+ ref = copy.deepcopy(net)+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)+ x = torch.randn(4, 16, device = "cuda")+ cg.capture_hold() # replica-side work that does not drain+ try:+ with torch.inference_mode():+ for _ in range(3):+ assert torch.equal(net(x), ref(x))+ assert handle.stats["captures"] == 0 and handle.stats["fallbacks"] == 0+ assert not any(g.poisoned for g in handle.graphs)+ assert cg._CAPTURE_DEPTH == 0+ finally:+ cg.capture_release()+ with torch.inference_mode():+ for _ in range(3):+ assert torch.equal(net(x), ref(x))+ assert handle.stats["captures"] > 0 and handle.stats["fallbacks"] == 0+ assert handle.stats["replays"] > 0+ handle.free()diff --git a/studio/backend/tests/test_diffusion_cfg_parallel_tier.py b/studio/backend/tests/test_diffusion_cfg_parallel_tier.pynew file mode 100644index 00000000000..564d50fb961--- /dev/null+++ b/studio/backend/tests/test_diffusion_cfg_parallel_tier.py@@ -0,0 +1,906 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CFG-parallel on the default speed tier and on Z-Image, CPU only (the replica shares the CPU device).++The deferred speed profile arms a whole-step CUDA graph on the denoiser's forward slot, where the CFG-parallel router+sits: the graph must install under the router so the guidance branches keep routing. Z-Image hands its split guided+call to the router as an explicit pair, since its halves are fresh tensors that identity pairing never matches."""++from __future__ import annotations++import threading+import types++import pytest+import torch++from core.inference import diffusion_cuda_graph as cg+from core.inference import diffusion_multigpu as mg++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)+ monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)+ yield+++class _Dit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4)++ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ return (self.proj(hidden_states) * timestep + encoder_hidden_states.sum(),)+++def _router(dit):+ replicas = []++ def factory():+ r = mg.build_replica(dit, torch.device("cpu"))+ replicas.append(r)+ return r++ router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu", replica_factory = factory)+ dit.forward = router+ dit._unsloth_cfg_router = router+ return router, replicas+++def _cfg_loop(dit, latents, steps):+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))+ with torch.no_grad():+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ a = dit(hidden_states = latents, timestep = t, encoder_hidden_states = cond)[0]+ b = dit(hidden_states = latents, timestep = t, encoder_hidden_states = uncond)[0]+ latents = latents - 0.1 * (b + 4.0 * (a - b))+ return latents+++class _Pipe:+ def __init__(self, transformer):+ self.transformer = transformer+++def test_graph_wrapper_installs_under_the_router_and_branches_keep_routing():+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 5)+ router, replicas = _router(dit)+ pipe = _Pipe(dit)+ mg._CFG_ROUTERS[pipe] = router+ # what the deferred speed profile arms on the default tier (not enabled here: a CPU tensor is never captured)+ graph = cg.GraphedForward(dit).install()+ assert dit.__dict__["forward"] is router and router.inner is graph+ assert not mg.drop_displaced_cfg_router(pipe, "test")+ out = _cfg_loop(dit, latents.clone(), steps = 5)+ assert torch.equal(out, reference)+ assert router.stats["steps_parallel"] == 4 and router.stats["replica_calls"] == 4+ assert len(replicas) == 1 and router.disabled_reason is None+ assert "forward" not in replicas[0].__dict__+ graph.uninstall()+ assert router.inner is None and dit.__dict__["forward"] is router+ graph.install()+ mg.uninstall_cfg_parallel(pipe)+ assert dit.__dict__["forward"] is graph+++def test_primary_branch_replays_through_the_inner_graph(monkeypatch):+ torch.manual_seed(0)+ dit = _Dit()+ router, _replicas = _router(dit)+ graph = cg.GraphedForward(dit).install()+ seen = []+ eager = graph._eager++ def counting(args, kwargs):+ seen.append(kwargs["encoder_hidden_states"])+ return eager(args, kwargs)++ monkeypatch.setattr(graph, "_eager", counting)+ _cfg_loop(dit, torch.randn(2, 4), steps = 4)+ # step 0 runs both branches on the source (the replica is built at its odd branch), then only the even ones+ assert len(seen) == 4 + 1+ assert router.stats["steps_parallel"] == 3+++def test_replica_jobs_and_captures_never_overlap():+ entered = threading.Event()+ release = threading.Event()+ captured = threading.Event()++ def replica_job():+ with cg.outside_capture():+ entered.set()+ release.wait(5)++ def capture():+ with cg._capturing():+ captured.set()++ a = threading.Thread(target = replica_job)+ a.start()+ assert entered.wait(5)+ b = threading.Thread(target = capture)+ b.start()+ assert not captured.wait(0.3) # the capture waits for the replica forward+ release.set()+ assert captured.wait(5)+ a.join(5)+ b.join(5)++ # and the other way round: a replica job waits for a recording to end+ recording = threading.Event()+ stop = threading.Event()+ ran = threading.Event()++ def long_capture():+ with cg._capturing():+ recording.set()+ stop.wait(5)++ def job():+ with cg.outside_capture():+ ran.set()++ c = threading.Thread(target = long_capture)+ c.start()+ assert recording.wait(5)+ d = threading.Thread(target = job)+ d.start()+ assert not ran.wait(0.3)+ stop.set()+ assert ran.wait(5)+ c.join(5)+ d.join(5)+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0+++def test_a_replica_thread_that_records_does_not_wait_on_itself():+ done = threading.Event()++ def job():+ with cg.outside_capture():+ with cg._capturing():+ done.set()++ t = threading.Thread(target = job)+ t.start()+ assert done.wait(5)+ t.join(5)+++zmod = None+++def _zimage():+ global zmod+ if zmod is None:+ import importlib+ try:+ zmod = importlib.import_module("diffusers.models.transformers.transformer_z_image")+ except (ImportError, RuntimeError) as exc:+ pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")+ torch.manual_seed(0)+ return zmod.ZImageTransformer2DModel(+ all_patch_size = (2,),+ all_f_patch_size = (1,),+ in_channels = 16,+ dim = 32,+ n_layers = 2,+ n_refiner_layers = 1,+ n_heads = 2,+ n_kv_heads = 2,+ norm_eps = 1e-5,+ qk_norm = True,+ cap_feat_dim = 16,+ rope_theta = 256.0,+ t_scale = 1000.0,+ axes_dims = [8, 4, 4],+ axes_lens = [512, 64, 64],+ ).eval()+++def _zimage_steps(model, pipe, steps):+ from core.inference import diffusion_zimage_cfg_split as zs++ outs = []+ g = torch.Generator().manual_seed(7)+ latent = torch.randn(16, 1, 8, 8, generator = g)+ cond = torch.randn(9, 16, generator = g)+ neg = torch.randn(40, 16, generator = g)+ with torch.no_grad(), zs.render(pipe, 1):+ for i in range(steps):+ # what ZImagePipeline passes on a guided step: fresh repeated latents and timesteps every call+ t = torch.tensor([1.0 - i / steps]).repeat(2)+ out = model([latent.clone(), latent.clone()], t, [cond, neg], return_dict = False)[0]+ latent = latent - 0.1 * (out[1] + 3.0 * (out[0] - out[1]))+ outs.append(latent)+ return outs+++def _zs():+ from core.inference import diffusion_zimage_cfg_split as zs+ yield zs+ zs.uninstall_zimage_cfg_split()+++def test_zimage_split_runs_its_halves_as_one_parallel_pair(_zs):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ assert _zs.install_zimage_cfg_split(pipe, offload_policy = "none") is None+ reference = _zimage_steps(model, pipe, 4)+ router, replicas = _router(model)+ out = _zimage_steps(model, pipe, 4)+ for a, b in zip(out, reference):+ assert torch.equal(a, b)+ assert len(replicas) == 1 and router.disabled_reason is None+ assert router.stats["pairs_parallel"] == 4+ assert router.stats["replica_calls"] == 4 and router.stats["steps_parallel"] == 4+ assert router.stats["primary_calls"] == 4+ status = _zs.zimage_cfg_split_status(pipe)+ assert status["split"] == 8 and status["batched"] == 0 and status["parallel"] == 4+++def test_zimage_pair_falls_back_to_the_source_when_the_replica_fails(_zs):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ reference = _zimage_steps(model, pipe, 3)+ router, replicas = _router(model)++ def broken():+ raise RuntimeError("replica out of memory")++ router.replica_factory = broken+ out = _zimage_steps(model, pipe, 3)+ for a, b in zip(out, reference):+ assert torch.equal(a, b)+ assert router.disabled_reason is not None and router.stats["pairs_parallel"] == 0+++def test_a_graph_armed_at_load_moves_under_a_later_router(monkeypatch):+ # speed tier armed at load (no deferral): the graph is in the slot first, then install_cfg_parallel runs+ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ graph = cg.GraphedForward(dit).install()+ assert mg._hooks_present(dit) is None+ probes = {+ o: mg.DeviceProbe(ordinal = o, name = "T4", total_mib = 15000, free_mib = 14000) for o in (0, 1)+ }+ monkeypatch.setattr(mg, "cfg_parallel_decline", lambda pipe, **kw: None)+ monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))+ monkeypatch.setattr(+ mg, "probe_devices", lambda ordinals, **kw: {o: probes[o] for o in ordinals}+ )+ pipe = _Pipe(dit)+ try:+ with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):+ assert mg.install_cfg_parallel(pipe, offload_policy = "none") is None+ router = mg._CFG_ROUTERS[pipe]+ assert (+ dit.__dict__["forward"] is router and router.inner is graph and not router.had_override+ )+ router.primary = router.secondary = torch.device("cpu")+ router.replica_factory = lambda: mg.build_replica(dit, torch.device("cpu"))+ out = _cfg_loop(dit, latents.clone(), steps = 4)+ assert torch.equal(out, reference)+ assert router.stats["steps_parallel"] == 3 and router.disabled_reason is None+ finally:+ mg.uninstall_cfg_parallel(pipe)+ assert dit.__dict__["forward"] is graph+++def test_a_queued_replica_job_holds_captures_off_until_it_has_dropped_its_inputs():+ worker = mg._ReplicaWorker(torch.device("cpu"))+ worker.call(lambda: None) # thread started and bound+ assert cg._OUTSIDE_JOBS == 0+ gate = threading.Event()+ dropped = []++ class Input:+ def __del__(self):+ dropped.append(cg._OUTSIDE_JOBS)++ def job(x = Input()):+ gate.wait(5)+ return 1++ done = worker.submit(job)+ del job+ assert cg._OUTSIDE_JOBS == 1+ captured = threading.Event()++ def capture():+ with cg._capturing():+ captured.set()++ t = threading.Thread(target = capture)+ t.start()+ assert not captured.wait(0.3)+ gate.set()+ assert mg._ReplicaWorker.wait(done) == 1+ assert captured.wait(5)+ t.join(5)+ # the job's inputs were released while the hold was still counted+ assert dropped == [1]+ assert cg._OUTSIDE_JOBS == 0+ worker.close()+++def test_a_graph_installs_under_a_router_nested_in_a_static_step_skip():+ # deferred speed: the auto static skip wraps the router armed at load, then the graph arms+ from core.inference.diffusion_step_skip import StaticStepSkip++ torch.manual_seed(0)+ dit = _Dit()+ latents = torch.randn(2, 4)+ reference = _cfg_loop(dit, latents.clone(), steps = 4)+ router, _replicas = _router(dit)+ pipe = _Pipe(dit)+ mg._CFG_ROUTERS[pipe] = router+ skip = StaticStepSkip(dit, inner = router)+ dit.__dict__["forward"] = skip+ graph = cg.GraphedForward(dit).install()+ assert dit.__dict__["forward"] is skip and skip.inner is router and router.inner is graph+ graph.uninstall()+ assert router.inner is None and skip.inner is router+ graph.install()+ # a replica cannot carry the skip: the router is dropped and the skip keeps the graph+ assert mg.drop_displaced_cfg_router(pipe, "test")+ assert mg._CFG_DECLINES[pipe] == "a static step skip wraps the denoiser"+ assert pipe not in mg._CFG_ROUTERS+ assert dit.__dict__["forward"] is skip and skip.inner is graph and router.inner is None+ assert torch.equal(_cfg_loop(dit, latents.clone(), steps = 4), reference)+++def test_a_worker_that_fails_to_start_releases_its_capture_hold(monkeypatch):+ assert cg._OUTSIDE_JOBS == 0++ def refuse(self):+ raise RuntimeError("can't start new thread")++ monkeypatch.setattr(threading.Thread, "start", refuse)+ with pytest.raises(RuntimeError):+ mg._ReplicaWorker(torch.device("cpu"))+ assert cg._OUTSIDE_JOBS == 0+++def test_captures_stay_held_off_while_a_replica_result_is_copied_home(_zs, monkeypatch):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ router, _replicas = _router(model)+ held = []+ move = mg.move_to_device++ def recording(obj, device, *a, **kw):+ if device == router.primary:+ held.append(cg._OUTSIDE_JOBS)+ return move(obj, device, *a, **kw)++ monkeypatch.setattr(mg, "move_to_device", recording)+ _zimage_steps(model, pipe, 3)+ assert router.stats["pairs_parallel"] == 3+ assert held and all(n >= 1 for n in held)+ assert cg._OUTSIDE_JOBS == 0+++def test_captures_stay_held_off_while_replica_inputs_are_copied(_zs, monkeypatch):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ router, _replicas = _router(model)+ to_replica = mg.CfgParallelRouter._to_replica+ held = []++ def recording(self, obj):+ real = cg.capture_hold++ def hold():+ real()+ held.append(cg._OUTSIDE_JOBS)++ monkeypatch.setattr(cg, "capture_hold", hold)+ try:+ return to_replica(self, obj)+ finally:+ monkeypatch.setattr(cg, "capture_hold", real)++ monkeypatch.setattr(mg.CfgParallelRouter, "_to_replica", recording)+ _zimage_steps(model, pipe, 2)+ assert router.stats["pairs_parallel"] == 2+ assert len(held) == 4 and all(n >= 1 for n in held)+ assert cg._OUTSIDE_JOBS == 0+++def test_a_failed_replica_input_copy_falls_back_to_the_source(_zs, monkeypatch):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ router, _replicas = _router(model)++ def boom(self, obj):+ raise RuntimeError("CUDA out of memory on the replica card")++ monkeypatch.setattr(mg.CfgParallelRouter, "_to_replica", boom)+ _zimage_steps(model, pipe, 1)+ assert router.disabled_reason and "replica input copy failed" in router.disabled_reason+ assert router.stats["pairs_parallel"] == 0+ assert cg._OUTSIDE_JOBS == 0+++def test_a_failed_result_copy_home_recomputes_the_second_half_on_the_source(_zs, monkeypatch):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ plain = types.SimpleNamespace(transformer = _zimage())+ _zs.install_zimage_cfg_split(plain, offload_policy = "none")+ expected = _zimage_steps(plain.transformer, plain, 1)+ router, _replicas = _router(model)++ def boom(out, primary):+ raise RuntimeError("peer copy failed")++ monkeypatch.setattr(mg, "_to_primary", boom)+ got = _zimage_steps(model, pipe, 1)+ assert router.disabled_reason and "replica call failed" in router.disabled_reason+ assert router.stats["pairs_parallel"] == 0+ assert all(torch.equal(a, b) for a, b in zip(got, expected))+++def test_the_replica_is_built_with_captures_held_off(_zs, monkeypatch):+ model = _zimage()+ pipe = types.SimpleNamespace(transformer = model)+ _zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ router, replicas = _router(model)+ factory = router.replica_factory+ held = []++ def building():+ held.append(cg._OUTSIDE_JOBS)+ return factory()++ router.replica_factory = building+ _zimage_steps(model, pipe, 1)+ assert held and held[0] >= 1+ assert cg._OUTSIDE_JOBS == 0+++class _FakeCopy:+ """A carried input on another card: ``to`` records whether a capture hold was up, or fails like a full card."""++ device = torch.device("cuda", 0)++ def __init__(self, fail = False):+ self.fail = fail+ self.held = []++ def to(self, device):+ self.held.append((cg._OUTSIDE_JOBS, getattr(cg._OUTSIDE_LOCAL, "depth", 0)))+ if self.fail:+ raise RuntimeError("CUDA out of memory on the replica card")+ return self+++def _bare_router():+ dit = _Dit()+ return mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu", replica_factory = lambda: None)+++def test_prefetch_copies_run_with_captures_held_off():+ router = _bare_router()+ carried = _FakeCopy()+ router.carry = [carried]+ router._prefetch((), {})+ assert carried.held == [(1, 1)]+ assert router.prefetched[id(carried)] == (carried, carried)+ assert cg._OUTSIDE_JOBS == 0+++def test_a_failed_prefetch_copy_is_harmless():+ router = _bare_router()+ ok, bad = _FakeCopy(), _FakeCopy(fail = True)+ router.carry = [ok, bad]+ router._prefetch((), {}) # must not raise: the call copies its own inputs later+ assert bad.held and not router.prefetched+ assert cg._OUTSIDE_JOBS == 0+++def test_a_prefetch_refused_by_a_long_recording_makes_no_copy(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ router = _bare_router()+ carried = _FakeCopy()+ router.carry = [carried]+ recording, stop = threading.Event(), threading.Event()++ def long_capture():+ with cg._capturing():+ recording.set()+ stop.wait(5)++ t = threading.Thread(target = long_capture)+ t.start()+ assert recording.wait(5)+ try:+ router._prefetch((), {})+ finally:+ stop.set()+ t.join(5)+ assert carried.held == [] and not router.prefetched+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0+++def test_a_capture_that_outwaits_replica_work_is_refused_not_recorded(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ cg.capture_hold() # replica-side work that never drains (this thread has no held_work of its own)+ try:+ ran = []+ with pytest.raises(cg.CaptureRefused):+ with cg._capturing():+ ran.append(cg._CAPTURE_DEPTH)+ assert ran == [] and cg._CAPTURE_DEPTH == 0+ assert cg.capture_refused_now()+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 30.0)+ with pytest.raises(cg.CaptureRefused):+ with cg._capturing():+ ran.append(1)+ assert ran == []+ finally:+ cg.capture_release()+ assert not cg.capture_refused_now()+ with cg._capturing():+ ran.append(cg._CAPTURE_DEPTH)+ assert ran == [1] and cg._CAPTURE_DEPTH == 0+++def test_replica_work_that_outwaits_a_recording_is_refused_and_never_runs(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ recording, stop = threading.Event(), threading.Event()++ def long_capture():+ with cg._capturing():+ recording.set()+ stop.wait(5)++ t = threading.Thread(target = long_capture)+ t.start()+ assert recording.wait(5)+ ran = []+ try:+ with pytest.raises(cg.OutsideWorkRefused):+ with cg.outside_capture():+ ran.append(1)+ assert ran == [] and cg._OUTSIDE_JOBS == 0+ finally:+ stop.set()+ t.join(5)+ with cg.outside_capture():+ ran.append(1)+ assert ran == [1] and cg._OUTSIDE_JOBS == 0+++def test_a_refused_worker_job_reports_an_error_and_the_worker_keeps_serving(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ recording, stop = threading.Event(), threading.Event()++ def long_capture():+ with cg._capturing():+ recording.set()+ stop.wait(5)++ t = threading.Thread(target = long_capture)+ t.start()+ assert recording.wait(5)+ ran = []+ try:+ # started while the capture records: the start-up bind is refused, the thread lives on+ worker = mg._ReplicaWorker(torch.device("cpu"))+ with pytest.raises(cg.OutsideWorkRefused):+ worker.call(lambda: ran.append(1))+ assert ran == [] and cg._OUTSIDE_JOBS == 0+ finally:+ stop.set()+ t.join(5)+ assert worker.call(lambda: 7) == 7+ assert cg._OUTSIDE_JOBS == 0+ worker.close()+++def _long_capture():+ recording, stop = threading.Event(), threading.Event()++ def run():+ with cg._capturing():+ recording.set()+ stop.wait(10)++ t = threading.Thread(target = run)+ t.start()+ assert recording.wait(5)++ def end():+ stop.set()+ t.join(5)++ return end+++def test_a_replica_build_refused_by_a_recording_makes_no_allocator_call(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ flushes = []+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: flushes.append(cg.capture_in_progress()))+ router, replicas = _router(_Dit())+ end = _long_capture()+ try:+ router._build_replica()+ finally:+ end()+ assert router.replica is None and replicas == []+ assert "replica unavailable" in router.disabled_reason+ # the capture was still recording: no empty_cache while it ran+ assert flushes == []+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0+++def test_a_failed_replica_build_flushes_with_captures_held_off(monkeypatch):+ flushes = []+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: flushes.append(cg._OUTSIDE_JOBS))+ router, _replicas = _router(_Dit())++ def boom():+ raise RuntimeError("CUDA out of memory on the replica card")++ router.replica_factory = boom+ router._build_replica()+ assert "replica unavailable" in router.disabled_reason+ assert flushes == [1] and cg._OUTSIDE_JOBS == 0+++def _step(dit, latents, t, cond, uncond):+ a = dit(hidden_states = latents, timestep = t, encoder_hidden_states = cond)[0]+ b = dit(hidden_states = latents, timestep = t, encoder_hidden_states = uncond)[0]+ return latents - 0.1 * (b + 4.0 * (a - b))+++def test_a_refused_prefetch_sends_the_rest_of_the_step_to_the_source_without_waiting_again(+ monkeypatch,+):+ torch.manual_seed(0)+ dit = _Dit()+ cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))+ uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))+ ts = [torch.tensor(1.0 - i / 5) for i in range(5)]+ with torch.no_grad():+ ref = torch.randn(2, 4, generator = torch.Generator().manual_seed(3))+ latents = ref.clone()+ refs = []+ for t in ts:+ ref = _step(dit, ref, t, cond, uncond)+ refs.append(ref)+ router, _replicas = _router(dit)+ for t in ts[:2]:+ latents = _step(dit, latents, t, cond, uncond)+ assert router.stats["steps_parallel"] == 1 and router.replica is not None+ waits = []+ real = cg.held_work++ def counted():+ waits.append(cg.capture_in_progress())+ return real()++ monkeypatch.setattr(cg, "held_work", counted)+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.3)+ end = _long_capture()+ try:+ for t in ts[2:4]:+ latents = _step(dit, latents, t, cond, uncond)+ finally:+ end()+ # one wait on the recording (the first prefetch); every later replica attempt went to the source at once+ assert waits == [True]+ assert router.disabled_reason is None+ assert router.stats["steps_parallel"] == 1 and router.stats["exclusion_skips"] >= 2+ latents = _step(dit, latents, ts[4], cond, uncond)+ assert router.stats["steps_parallel"] == 2 and not router.exclusion_refused+ assert torch.allclose(latents, refs[-1], atol = 1e-6)+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0+++def test_a_held_capture_still_waits_for_another_threads_held_work(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 5.0)+ other_in, other_go = threading.Event(), threading.Event()++ def other_worker():+ with cg.outside_capture():+ other_in.set()+ other_go.wait(5)++ o = threading.Thread(target = other_worker)+ o.start()+ assert other_in.wait(5)+ captured = threading.Event()++ def recording_worker():+ with cg.outside_capture():+ with cg._capturing():+ captured.set()++ r = threading.Thread(target = recording_worker)+ r.start()+ try:+ # the other worker's held work is still running: this worker's own hold is discounted, the other's is not+ assert not captured.wait(0.3)+ finally:+ other_go.set()+ assert captured.wait(5)+ o.join(5)+ r.join(5)+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0 and cg._OUTSIDE_PARKED == [0]+ assert not cg._OUTSIDE_STUCK[0]+++def test_a_held_capture_alone_records_at_once_and_two_record_one_after_the_other(monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 5.0)+ with cg.outside_capture():+ assert not cg.capture_refused_now()+ with cg._capturing(): # only this thread's own hold is up: no self-deadlock+ assert cg._CAPTURE_DEPTH == 1+ both_in = threading.Barrier(2)+ overlap, errors = [], []+ live = [0]+ lock = threading.Lock()++ def worker():+ try:+ with cg.outside_capture():+ both_in.wait(5)+ with cg._capturing():+ with lock:+ live[0] += 1+ overlap.append(live[0])+ threading.Event().wait(0.05)+ with lock:+ live[0] -= 1+ except BaseException as exc: # noqa: BLE001+ errors.append(exc)++ threads = [threading.Thread(target = worker) for _ in range(2)]+ t0 = __import__("time").perf_counter()+ for t in threads:+ t.start()+ for t in threads:+ t.join(10)+ # both parked holders do not wait each other out: one records, then the other, well inside the wait+ assert errors == [] and overlap == [1, 1]+ assert __import__("time").perf_counter() - t0 < 2.0+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0 and cg._OUTSIDE_PARKED == [0]+++class _FxTrace:+ """torch <= 2.13's process-wide FX tracing flag; ``waiter``'s warm calls wait for a compile to start, forcing the overlap."""++ def __init__(+ self,+ compiled = (),+ waiter = None,+ ):+ self.owner = None+ self.compiled = set(compiled)+ self.waiter = waiter+ self.started = threading.Event()+ self.entered = threading.Event()+ self.errors: list = []++ def enter(self, side, key):+ if side == self.waiter and key in self.compiled:+ self.started.wait(0.5)+ if self.owner not in (None, threading.get_ident()):+ self.entered.set()+ self.errors.append((side, key))+ raise RuntimeError(+ "Detected that you are using FX to symbolically trace a dynamo-optimized function. "+ "This is not supported at the moment."+ )+ if key in self.compiled:+ return+ self.owner = threading.get_ident()+ self.started.set()+ self.entered.wait(0.3)+ self.compiled.add(key)+ self.owner = None+++_FX: list = [None]+++class _TracedDit(_Dit):+ def forward(+ self,+ hidden_states,+ timestep,+ encoder_hidden_states,+ return_dict = False,+ ):+ side = (+ "replica"+ if threading.current_thread().name.startswith("unsloth-cfg-replica")+ else "primary"+ )+ key = (side, tuple(hidden_states.shape), tuple(encoder_hidden_states.shape))+ _FX[0].enter(side, key)+ return super().forward(hidden_states, timestep, encoder_hidden_states, return_dict)+++def _expected(dit, x, t, enc):+ return _Dit.forward(dit, hidden_states = x, timestep = t, encoder_hidden_states = enc)[0]+++def _pairs(dit, router, x, cond, uncond, steps):+ outs = []+ with torch.no_grad():+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps)+ kw = dict(hidden_states = x, timestep = t, encoder_hidden_states = uncond)+ pair = router.run_pair(+ lambda: dit(hidden_states = x, timestep = t, encoder_hidden_states = cond), (), kw+ )+ assert pair is not None and pair[1] is not None, router.disabled_reason+ assert torch.equal(pair[0][0], _expected(dit, x, t, cond))+ assert torch.equal(pair[1][0], _expected(dit, x, t, uncond))+ outs.append(pair)+ return outs+++def test_a_pair_new_to_the_replica_never_overlaps_the_primary_half():+ torch.manual_seed(0)+ dit = _TracedDit()+ x, cond, uncond = torch.randn(2, 4), torch.randn(3, 4), torch.randn(5, 4)+ _FX[0] = _FxTrace(compiled = {("primary", (2, 4), (3, 4))}, waiter = "primary")+ router, _replicas = _router(dit)+ _pairs(dit, router, x, cond, uncond, 3)+ assert _FX[0].errors == [] and router.disabled_reason is None+ assert ("replica", (2, 4), (5, 4)) in _FX[0].compiled+ assert router.stats["pairs_parallel"] == 3+++def test_a_pair_new_to_the_primary_never_overlaps_the_replica_half():+ torch.manual_seed(0)+ dit = _TracedDit()+ x, uncond = torch.randn(2, 4), torch.randn(5, 4)+ _FX[0] = _FxTrace(compiled = {("primary", (2, 4), (3, 4)), ("replica", (2, 4), (5, 4))})+ router, _replicas = _router(dit)+ _pairs(dit, router, x, torch.randn(3, 4), uncond, 2)+ _FX[0].waiter = "replica"+ _pairs(dit, router, x, torch.randn(7, 4), uncond, 2) # a new primary-side length: it compiles+ assert _FX[0].errors == [] and router.disabled_reason is None+ assert ("primary", (2, 4), (7, 4)) in _FX[0].compiled+ assert router.stats["pairs_parallel"] == 4+++def test_a_prediction_new_to_the_replica_is_not_launched_beside_the_primary_branch():+ torch.manual_seed(0)+ dit = _TracedDit()+ latents = torch.randn(2, 4)+ _FX[0] = _FxTrace(compiled = {("primary", (2, 4), (3, 4))})+ reference = _cfg_loop(dit, latents.clone(), steps = 5)+ _FX[0].waiter = "primary"+ router, _replicas = _router(dit)+ out = _cfg_loop(dit, latents.clone(), steps = 5)+ assert torch.equal(out, reference)+ assert _FX[0].errors == [] and router.disabled_reason is None+ assert router.stats["replica_calls"] == 4 and router.stats["predicted_hits"] == 3diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.pyindex 2408ba0167f..b809f770404 100644--- a/studio/backend/tests/test_diffusion_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_cuda_graph.py@@ -14,6 +14,7 @@ import contextlibimport inspectimport jsonimport sys+import threadingimport typesimport weakref@@ -1617,3 +1618,60 @@ def test_eager_as_recorded_runs_the_original_call_when_the_copies_do_not_fit(monassert torch.equal(got, hidden + txt.mean())finally:cg.uninstall_all([handle])+++def test_a_refused_capture_runs_eager_and_records_on_a_later_call(stub_torch, monkeypatch):+ monkeypatch.setattr(cg, "OUTSIDE_WAIT_S", 0.05)+ retired = []+ real_retire = cg.retire_failed_capture+ monkeypatch.setattr(+ cg, "retire_failed_capture", lambda *a, **k: retired.append(a) or real_retire(*a, **k)+ )+ module = _FakeDiT()+ handle = _armed(module)+ cg.capture_hold() # replica-side work that does not drain+ try:+ handle(_t(), timestep = _t((1,)), return_dict = False)+ assert handle.poisoned is False and handle.stats["captures"] == 0+ assert handle.stats["capture_refusals"] == 1 and handle.stats["eager_calls"] == 1+ assert retired == [] and not handle.cache and cg._CAPTURE_DEPTH == 0+ calls = module.calls+ # while the count stands a capture is refused up front, warm-ups included+ handle(_t(), timestep = _t((1,)), return_dict = False)+ assert handle.stats["capture_refusals"] == 2 and module.calls == calls + 1+ finally:+ cg.capture_release()+ calls = module.calls+ handle(_t(), timestep = _t((1,)), return_dict = False)+ assert handle.poisoned is False and handle.stats["captures"] == 1+ assert module.calls == calls + cg.WARMUP_ITERS + 1+ assert retired == []+++def test_capture_warmups_wait_for_held_replica_work(stub_torch):+ seen = []++ class Dit(_FakeDiT):+ def forward(self, *args, **kwargs):+ seen.append(cg._OUTSIDE_JOBS)+ return super().forward(*args, **kwargs)++ handle = _armed(Dit())+ held, release = threading.Event(), threading.Event()++ def replica_job():+ with cg.outside_capture():+ held.set()+ release.wait(5)++ worker = threading.Thread(target = replica_job)+ worker.start()+ assert held.wait(5)+ timer = threading.Timer(0.3, release.set)+ timer.start()+ handle(_t(), timestep = _t((1,)), return_dict = False)+ worker.join(5)+ timer.join(5)+ assert handle.stats["captures"] == 1+ assert seen == [0] * (cg.WARMUP_ITERS + 1)+ assert cg._OUTSIDE_JOBS == 0 and cg._CAPTURE_DEPTH == 0diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.pyindex 31891fcc68c..26233d9979f 100644--- a/studio/backend/tests/test_diffusion_multigpu.py+++ b/studio/backend/tests/test_diffusion_multigpu.py@@ -964,7 +964,8 @@ def test_predicted_branch_hits_and_stays_exact():router, _replicas = _router(dit)out = _cfg_loop(dit, latents.clone(), steps = 6)assert torch.equal(out, reference)- assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0+ # the replica's first call (it may compile) runs unpredicted, after the primary branch+ assert router.stats["predicted_hits"] == 4 and router.stats["predicted_misses"] == 0assert router.stats["replica_calls"] == 5@@ -989,7 +990,7 @@ def test_mispredicted_branch_is_recomputed_exactly():router, _replicas = _router(dit)out = loop(5)assert torch.equal(out, reference)- assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 3assert router.stats["replica_calls"] == 4@@ -1013,7 +1014,7 @@ def test_inline_recomputed_scalars_are_predicted_by_value():router, _replicas = _router(dit)out = loop(6)assert torch.equal(out, reference)- assert router.stats["predicted_hits"] == 5 and router.stats["predicted_misses"] == 0+ assert router.stats["predicted_hits"] == 4 and router.stats["predicted_misses"] == 0def test_bitwise_equal_on_an_expanded_scalar():@@ -1171,7 +1172,7 @@ def test_in_place_edit_after_launch_rejects_the_prediction():router, _replicas = _router(dit)out = loop(5)assert torch.equal(out, reference)- assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 4+ assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 3router.prefetched = {id(uncond): (uncond, uncond)}router.pending = {"leaves": [],diff --git a/studio/backend/tests/test_diffusion_multigpu_cuda.py b/studio/backend/tests/test_diffusion_multigpu_cuda.pyindex ab25ed31e77..f8b702c1809 100644--- a/studio/backend/tests/test_diffusion_multigpu_cuda.py+++ b/studio/backend/tests/test_diffusion_multigpu_cuda.py@@ -102,3 +102,105 @@ def test_an_edited_buffer_rebuilds_the_replica():torch.cuda.synchronize()assert router.stats["replica_rebuilds"] >= 1assert torch.equal(out, reference)+++def _graphed_render(dit, latents, cond, neg, steps, renders):+ outs = []+ for _ in range(renders):+ x = latents.clone()+ for i in range(steps):+ t = torch.tensor(1.0 - i / steps, device = x.device)+ a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond, return_dict = False)[0]+ b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg, return_dict = False)[0]+ x = x - 0.1 * (b + 4.0 * (a - b))+ outs.append(x)+ router = dit.__dict__.get("_unsloth_cfg_router")+ if router is not None:+ router.end_render()+ torch.cuda.synchronize()+ return outs+++def test_default_tier_graph_under_the_router_replays_and_stays_exact():+ from core.inference import diffusion_cuda_graph as cg++ dit, latents, cond, neg = _setup()+ with torch.no_grad():+ single = cg.GraphedForward(dit).enable()+ reference = _graphed_render(dit, latents, cond, neg, 8, 3)+ assert single.stats["captures"] >= 1 and single.stats["replays"] > 0+ single.disable()+ router = _arm(dit)+ dit._unsloth_cfg_router = router+ # the deferred speed profile arms its graph after the router+ graph = cg.GraphedForward(dit).enable()+ assert dit.__dict__["forward"] is router and router.inner is graph+ out = _graphed_render(dit, latents, cond, neg, 8, 3)+ for a, b in zip(out, reference):+ assert torch.equal(a, b)+ assert graph.stats["captures"] >= 1 and graph.stats["replays"] > 0+ assert router.stats["steps_parallel"] > 0 and router.disabled_reason is None+ assert str(next(router.replica.parameters()).device) == "cuda:1"+ captures = graph.stats["captures"]+ with torch.no_grad():+ _graphed_render(dit, latents, cond * 2, neg, 8, 1)+ assert graph.stats["captures"] == captures # a follow-up prompt records nothing new+ graph.disable()+++def test_zimage_pair_runs_on_two_cards_and_stays_exact():+ zmod = pytest.importorskip("diffusers.models.transformers.transformer_z_image")+ from core.inference import diffusion_zimage_cfg_split as zs++ torch.manual_seed(0)+ model = (+ zmod.ZImageTransformer2DModel(+ all_patch_size = (2,),+ all_f_patch_size = (1,),+ in_channels = 16,+ dim = 32,+ n_layers = 2,+ n_refiner_layers = 1,+ n_heads = 2,+ n_kv_heads = 2,+ norm_eps = 1e-5,+ qk_norm = True,+ cap_feat_dim = 16,+ rope_theta = 256.0,+ t_scale = 1000.0,+ axes_dims = [8, 4, 4],+ axes_lens = [512, 64, 64],+ )+ .eval()+ .cuda(0)+ )++ class Pipe:+ transformer = model++ pipe = Pipe()+ g = torch.Generator().manual_seed(7)+ latent0 = torch.randn(16, 1, 8, 8, generator = g).cuda(0)+ cond = torch.randn(9, 16, generator = g).cuda(0)+ neg = torch.randn(40, 16, generator = g).cuda(0)++ def steps():+ latent = latent0.clone()+ with torch.no_grad(), zs.render(pipe, 1):+ for i in range(4):+ t = torch.tensor([1.0 - i / 4], device = "cuda:0").repeat(2)+ out = model([latent.clone(), latent.clone()], t, [cond, neg], return_dict = False)[0]+ latent = latent - 0.1 * (out[1] + 3.0 * (out[0] - out[1]))+ torch.cuda.synchronize()+ return latent++ try:+ zs.install_zimage_cfg_split(pipe, offload_policy = "none")+ reference = steps()+ router = _arm(model)+ model._unsloth_cfg_router = router+ out = steps()+ finally:+ zs.uninstall_zimage_cfg_split(pipe)+ assert torch.equal(out, reference)+ assert router.stats["pairs_parallel"] == 4 and router.disabled_reason is None