Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CFG-parallel: treat a changed Python number, storage offset or autocast as a cold call (#14614)

Daniel Hana07c510b6861
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index d5bb17d208b..c4828900b03 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -1717,21 +1717,38 @@ def _mirror_hook_state(pairs: list) -> tuple:
return state
+def _autocast_state(torch: Any) -> tuple:
+ try:
+ return (
+ torch.is_autocast_cache_enabled(),
+ *(
+ (dev, torch.get_autocast_dtype(dev))
+ if torch.is_autocast_enabled(dev)
+ else (dev, None)
+ for dev in ("cuda", "cpu")
+ ),
+ )
+ except Exception: # noqa: BLE001 - torch without per-device autocast queries
+ return ()
+
+
def _call_signature(args: tuple, kwargs: dict) -> tuple:
- """What a compiled forward's guards mostly key on: tensor layout, flags and modes (never per-step scalars)."""
+ """What a compiled forward's guards key on, numbers by value: 0 / 1 and ``dynamic=False`` compiles stay specialised."""
torch = _torch()
sig: list = []
for path, leaf in _leaves((args, kwargs)):
if isinstance(leaf, torch.Tensor):
try:
stride = tuple(leaf.stride())
+ offset = leaf.storage_offset()
except Exception: # noqa: BLE001 - layouts without strides
- stride = None
+ stride = offset = None
sig.append(
(
path,
tuple(leaf.shape),
stride,
+ offset,
leaf.dtype,
leaf.requires_grad,
leaf.is_inference(),
@@ -1739,9 +1756,16 @@ def _call_signature(args: tuple, kwargs: dict) -> tuple:
)
elif leaf is None or isinstance(leaf, (bool, str)):
sig.append((path, leaf))
+ elif isinstance(leaf, (int, float)):
+ sig.append((path, type(leaf), leaf))
else:
sig.append((path, type(leaf)))
- return tuple(sig), torch.is_inference_mode_enabled(), torch.is_grad_enabled()
+ return (
+ tuple(sig),
+ torch.is_inference_mode_enabled(),
+ torch.is_grad_enabled(),
+ _autocast_state(torch),
+ )
def _note_warm(seen: set, sig: tuple) -> None:
diff --git a/studio/backend/tests/test_diffusion_multigpu.py b/studio/backend/tests/test_diffusion_multigpu.py
index 26233d9979f..4a322f961b2 100644
--- a/studio/backend/tests/test_diffusion_multigpu.py
+++ b/studio/backend/tests/test_diffusion_multigpu.py
@@ -2366,3 +2366,51 @@ def test_replica_blocks_get_their_own_compile_guard():
guard = getattr(wrapped, "_unsloth_compile_guard", None)
assert guard is not None and guard is not source_guard
assert replica.__dict__.get("_unsloth_compile_guard") is guard
+
+
[email protected]("dynamic", [None, False])
+def test_a_warm_signature_never_compiles_on_a_changed_number(dynamic):
+ # A warm call runs beside the other side, so it must never compile (FX tracing is process-wide up to 2.13).
+ compiles = []
+
+ def backend(gm, _example_inputs):
+ compiles.append(1)
+ return gm.forward
+
+ @torch.compile(backend = backend, dynamic = dynamic)
+ def fn(x, scale, k, kw):
+ return x * scale + k + kw["s"]
+
+ x = torch.randn(4)
+ warm: set = set()
+ calls = [
+ (1.0, 1, 0.5),
+ (2.0, 1, 0.5),
+ (3.0, 1, 0.5),
+ (3.0, 2, 0.5),
+ (3.0, 3, 0.5),
+ (4.5, 7, 0.25),
+ (4.5, 0, 0.25),
+ (4.5, 1, 0.25),
+ (4.5, 7, 0.25),
+ (5.5, 1, 0.25),
+ ]
+ for scale, k, s in calls:
+ sig = mg._call_signature((x, scale, k), {"kw": {"s": s}})
+ before = len(compiles)
+ fn(x, scale, k, {"s": s})
+ if sig in warm:
+ assert len(compiles) == before, (scale, k, s)
+ mg._note_warm(warm, sig)
+ torch._dynamo.reset()
+
+
+def test_warm_signature_keys_on_storage_offset_and_autocast():
+ base = torch.zeros(8)
+ a = mg._call_signature((base[:4],), {})
+ assert mg._call_signature((base[4:],), {}) != a
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ assert mg._call_signature((base[:4],), {}) != a
+ with torch.autocast("cpu", enabled = False, cache_enabled = False):
+ assert mg._call_signature((base[:4],), {}) != a
+ assert mg._call_signature((base[:4],), {}) == a