Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fixes to XPU offloading during training (#9534)

Joseph Antonuccifac17ac642f5
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.py
index d025b9c63fd..83c4f880734 100644
--- a/studio/backend/core/training/diffusion_checkpoint.py
+++ b/studio/backend/core/training/diffusion_checkpoint.py
@@ -590,6 +590,9 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any
"python": _random_state_to_json(random.getstate()),
"streams": {},
"numpy": None,
+ # Which accelerator the torch_* device tensors below belong to. The resume preflight reads it: the device
+ # generator of one backend says nothing on another, and nothing else in the identity distinguishes them.
+ "accelerator": _rng_accelerator(),
}
for name, stream in (streams or {}).items():
try:
@@ -624,6 +627,14 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any
# (which only requires torch_cpu) offered it and the restore then left the CUDA generator freshly
# seeded, silently changing every latent and timestep draw.
tensors = {}
+ elif _xpu_available():
+ # The flow trainers draw randn_like on the training device, so an XPU run's noise stream lives in the XPU
+ # generator; capturing only torch_cpu resumed it from a fresh seed. Same all-or-nothing rule as CUDA.
+ try:
+ for i, state in enumerate(torch.xpu.get_rng_state_all()):
+ tensors[f"torch_xpu_{i}"] = state
+ except Exception: # noqa: BLE001 -- one device erroring loses the whole capture
+ tensors = {}
except Exception: # noqa: BLE001 -- torch RNG capture is best-effort
tensors = {}
return {"json": payload, "tensors": tensors}
@@ -685,10 +696,39 @@ def restore_rng_state(
if state is None:
continue
torch.cuda.set_rng_state(state.cpu().to(torch.uint8), i)
+ elif _xpu_available():
+ # Per device for the same reason as CUDA above: set_rng_state_all needs one state per visible device.
+ for i in range(torch.xpu.device_count()):
+ state = tensors.get(f"torch_xpu_{i}")
+ if state is None:
+ continue
+ torch.xpu.set_rng_state(state.cpu().to(torch.uint8), i)
except Exception: # noqa: BLE001 -- best-effort restore, never fatal
pass
+def _rng_accelerator() -> str:
+ """The backend whose device generator a capture holds: ``cuda``, ``xpu`` or ``cpu``."""
+ try:
+ import torch
+ if torch.cuda.is_available():
+ return "cuda"
+ except Exception: # noqa: BLE001 -- probe failure -> no CUDA generator captured
+ pass
+ return "xpu" if _xpu_available() else "cpu"
+
+
+def _xpu_available() -> bool:
+ """Guarded like ``resolve_train_device``'s probe: an uninitialised driver must not turn a
+ best-effort RNG capture into a raise."""
+ try:
+ import torch
+ fn = getattr(getattr(torch, "xpu", None), "is_available", None)
+ return bool(fn()) if callable(fn) else False
+ except Exception: # noqa: BLE001 -- probe failure -> no XPU generator to capture
+ return False
+
+
def _random_state_to_json(state: Any) -> Optional[list[Any]]:
"""``random.Random.getstate()`` is ``(version, tuple[int, ...], gauss)``; JSON has no tuples, so
store it as nested lists."""
@@ -1718,6 +1758,16 @@ def _assert_required_state(path: Path, manifest: dict[str, Any]) -> None:
isinstance(saved_streams.get(name), (list, tuple)) for name in _TRAINER_RNG_STREAMS
):
missing.append("the trainer's random-number streams")
+ # Same failure one level down: a CUDA bundle carries only torch_cuda_* keys, so resuming it on XPU (or the
+ # reverse) leaves the destination generator freshly seeded and reports a clean resume. Nothing else catches it,
+ # since the identity records the effective precision and that is bf16 on both. Only a KNOWN mismatch counts: a
+ # bundle predating this field must still resume.
+ saved_accel = rng_manifest.get("accelerator") if isinstance(rng_manifest, dict) else None
+ if isinstance(saved_accel, str) and saved_accel and saved_accel != _rng_accelerator():
+ missing.append(
+ f"the random-number state for this accelerator (written on {saved_accel}, "
+ f"resuming on {_rng_accelerator()})"
+ )
if not missing:
return
raise ResumeError(
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 7cb73740679..f6c349c6da3 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -57,8 +57,10 @@ from core.training.diffusion_train_common import (
discover_image_caption_pairs,
has_functional_torchao,
native_bf16_supported,
+ native_bf16_supported_xpu,
PermutationBatchSampler,
repo_is_prequantized,
+ resolve_train_device,
resolve_train_steps,
restore_resume_state,
write_resume_checkpoint,
@@ -268,13 +270,14 @@ def _bnb_4bit_config():
_repo_is_prequantized = repo_is_prequantized
-def _load_quantized_transformer(transformer_cls, cfg):
+def _load_quantized_transformer(transformer_cls, cfg, device):
"""Load ``cfg.base_model``'s transformer subfolder as a trainable nf4 QLoRA module."""
import torch
return transformer_cls.from_pretrained(
cfg.base_model,
subfolder = "transformer",
quantization_config = _bnb_4bit_config(),
+ device_map = {"": device},
torch_dtype = torch.bfloat16,
token = cfg.hf_token,
)
@@ -307,7 +310,7 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):
if base_precision == "nf4":
if not repo_is_prequantized(cfg.base_model):
- return _load_quantized_transformer(transformer_cls, cfg)
+ return _load_quantized_transformer(transformer_cls, cfg, device)
transformer = transformer_cls.from_pretrained(
cfg.base_model,
subfolder = "transformer",
@@ -1853,15 +1856,20 @@ def run_dit_lora_training(
save_on_stop = False
return True
- device = "cuda" if torch.cuda.is_available() else "cpu"
- # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since is_bf16_supported() counts
- # emulation.
+ device = resolve_train_device()
+ # bf16 throughout (fp32 on a CPU-only box, to keep import/unit tests architecture-agnostic). Both accelerator
+ # guards gate on NATIVE bf16, since is_bf16_supported() counts emulation on CUDA and on XPU alike.
if device == "cuda" and not native_bf16_supported():
raise ValueError(
"This trainer requires a bfloat16-capable GPU (Ampere or newer); "
"this CUDA device does not support bf16."
)
- weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32
+ if device == "xpu" and not native_bf16_supported_xpu():
+ raise ValueError(
+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "
+ "support bf16 natively."
+ )
+ weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32
_assert_trusted_base_model(cfg.base_model)
# Check the repo this run will FETCH: the canonical id would raise for a gated base already redirected to its
@@ -2002,6 +2010,8 @@ def _train_dit(
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
+ elif device == "xpu":
+ torch.xpu.empty_cache()
# The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.
if use_cache:
@@ -2043,6 +2053,8 @@ def _train_dit(
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
+ elif device == "xpu":
+ torch.xpu.empty_cache()
# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.
variant_rng = random.Random(cfg.seed + 1)
@@ -2209,8 +2221,8 @@ def _train_dit(
# bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles the fp32 LoRA
# params with the bnb 4-bit base matmuls.
autocast = (
- torch.autocast(device_type = "cuda", dtype = torch.bfloat16)
- if device == "cuda"
+ torch.autocast(device_type = device, dtype = torch.bfloat16)
+ if device in ("cuda", "xpu")
else nullcontext()
)
for opt_step in range(resumed, cfg.train_steps):
diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.py
index 1fa3360eb93..ac266ae0dde 100644
--- a/studio/backend/core/training/diffusion_h3_trainer.py
+++ b/studio/backend/core/training/diffusion_h3_trainer.py
@@ -83,6 +83,8 @@ from core.training.diffusion_train_common import (
_restore_perf_flags,
h3_train_unsupported_reason,
native_bf16_supported,
+ native_bf16_supported_xpu,
+ resolve_train_device,
resolve_train_steps,
train_recipe_overrides,
)
@@ -313,6 +315,7 @@ def _load_transformer(cfg, device, base_precision):
cfg.base_model,
subfolder = "transformer",
quantization_config = quant,
+ device_map = {"": device},
torch_dtype = torch.bfloat16,
token = cfg.hf_token,
cache_dir = cache_dir,
@@ -479,13 +482,18 @@ def run_h3_lora_training(
save_on_stop = False
return True
- device = "cuda" if torch.cuda.is_available() else "cpu"
+ device = resolve_train_device()
if device == "cuda" and not native_bf16_supported():
raise ValueError(
"This trainer requires a bfloat16-capable GPU (Ampere or newer); "
"this CUDA device does not support bf16."
)
- weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32
+ if device == "xpu" and not native_bf16_supported_xpu():
+ raise ValueError(
+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "
+ "support bf16 natively."
+ )
+ weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32
# allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local MiniMax-H3
# pipeline carries modular_model_index.json and no model_index.json, so the conventional shape
@@ -544,6 +552,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
+ elif device == "xpu":
+ torch.xpu.empty_cache()
# Phase 2: the clip cache. One canvas for the run, from the FIRST clip's aspect ratio: every other clip is
# cover-cropped onto it, so a mixed-aspect dataset trains on one geometry.
@@ -598,6 +608,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
+ elif device == "xpu":
+ torch.xpu.empty_cache()
# Phase 3: the denoiser.
base_precision = cfg.base_precision if cfg.base_precision != "auto" else "nf4"
@@ -670,8 +682,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
patch = tuple(transformer.config.patch_size)
index_sampler = PermutationBatchSampler(len(clip_paths), rng)
autocast = (
- torch.autocast(device_type = "cuda", dtype = torch.bfloat16)
- if device == "cuda"
+ torch.autocast(device_type = device, dtype = torch.bfloat16)
+ if device in ("cuda", "xpu")
else nullcontext()
)
stopped = False
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index c9316c24d59..372fe49fc19 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -613,10 +613,10 @@ def effective_mixed_precision(cfg: Any) -> str:
requested = str(getattr(cfg, "mixed_precision", "") or "")
if str(getattr(cfg, "resolved_family", "") or "").strip().lower() in _FLOW_TRAIN_FAMILIES:
- # No flow-matching trainer reads mixed_precision (weight_dtype is bf16 on CUDA, fp32 otherwise), so recording
- # the REQUEST failed a later bf16 resume as a precision mismatch between identical runs. Keyed on
- # _FLOW_TRAIN_FAMILIES so the answer follows the weight dtype.
- return "bf16" if torch.cuda.is_available() else "no"
+ # No flow trainer reads mixed_precision (weight_dtype is bf16 on an accelerator, fp32 otherwise), so recording
+ # the REQUEST failed a later bf16 resume as a mismatch between identical runs. Resolve the device the way the
+ # loops do, or an XPU run trains in bf16 and records "no".
+ return "bf16" if resolve_train_device() in ("cuda", "xpu") else "no"
if not torch.cuda.is_available():
return "no"
if requested == "bf16" and not native_bf16_supported():
@@ -644,6 +644,62 @@ def native_bf16_supported() -> bool:
return False
+def resolve_train_device() -> str:
+ """The device a flow-matching trainer runs on: CUDA first (so a box with both is unaffected),
+ then Intel XPU, else CPU.
+
+ Probes are guarded individually like ``dit_accelerator_missing_reason``: an uninitialised driver
+ whose ``is_available()`` raises must fall through, not kill the run. Shared so the two trainers
+ and the precision the run RECORDS cannot disagree."""
+ import torch # noqa: PLC0415 -- keep the import list light for the training subprocess
+
+ def _probe(module) -> bool:
+ try:
+ fn = getattr(module, "is_available", None)
+ return bool(fn()) if callable(fn) else False
+ except Exception: # noqa: BLE001 -- probe failure -> this accelerator is not usable
+ return False
+
+ if _probe(getattr(torch, "cuda", None)):
+ return "cuda"
+ if _probe(getattr(torch, "xpu", None)):
+ return "xpu"
+ return "cpu"
+
+
+def xpu_native_bf16_probe() -> Optional[bool]:
+ """Native-bf16 on the live XPU, or None when the capability cannot be determined. Never raises.
+
+ ``is_bf16_supported()`` defaults to ``including_emulation=True`` and short-circuits before
+ reading ``has_bfloat16_conversions``, so the bare call answers True for EVERY available XPU:
+ the same emulation trap ``native_bf16_supported`` avoids on the CUDA side. Ask explicitly.
+
+ Tri-state because ``get_device_properties()`` raises when no device is really there: collapsing
+ that to False made the pre-eviction preflight refuse nf4 on any host whose XPU cannot be
+ interrogated, a probe failing CLOSED."""
+ import torch # noqa: PLC0415
+
+ fn = getattr(getattr(torch, "xpu", None), "is_bf16_supported", None)
+ if not callable(fn):
+ return None
+ try:
+ return bool(fn(including_emulation = False))
+ except TypeError:
+ # A torch predating the including_emulation parameter: its answer is the only one there is.
+ try:
+ return bool(fn())
+ except Exception: # noqa: BLE001 -- unprobeable
+ return None
+ except Exception: # noqa: BLE001 -- unprobeable
+ return None
+
+
+def native_bf16_supported_xpu() -> bool:
+ """``xpu_native_bf16_probe`` collapsed for the trainer child, which runs ON the device: there an
+ unprobeable capability is a refusal, matching ``native_bf16_supported`` on the CUDA side."""
+ return xpu_native_bf16_probe() is True
+
+
def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:
"""Return a user-facing error string if ``resolved_family`` needs bf16 compute the live GPU
cannot provide, else None. The DiT trainer requires Ampere or newer and otherwise raises deep
@@ -661,6 +717,16 @@ def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:
"This trainer requires a bfloat16-capable GPU (Ampere or newer); this CUDA "
"device does not support bf16. Train the DiT families on a newer GPU."
)
+ # dit_accelerator_missing_reason accepts any available XPU, so without this the route
+ # admitted an emulation-only XPU, evicted the resident models, and only then hit the
+ # trainer's own guard: the eviction ordering this function exists to protect. Only a
+ # DEFINITE no rejects here; an unprobeable XPU is left to the child, since a preflight that
+ # fails closed would refuse hosts that train fine.
+ if resolve_train_device() == "xpu" and xpu_native_bf16_probe() is False:
+ return (
+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "
+ "support bf16 natively. Train the DiT families on a newer GPU."
+ )
except Exception: # noqa: BLE001 -- torch probe failure must not block a start
return None
return None
diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.py
index 0f4a9fe4be2..56c0a6ab167 100644
--- a/studio/backend/tests/test_diffusion_h3_trainer.py
+++ b/studio/backend/tests/test_diffusion_h3_trainer.py
@@ -1273,7 +1273,7 @@ def test_the_precision_recorded_for_h3_is_the_one_its_loop_runs_in():
# The claim the equality rests on: the same weight-dtype rule in both loops, and no reader of
# mixed_precision in the H3 one.
- dtype_rule = 'weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32'
+ dtype_rule = 'weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32'
h3_src = inspect.getsource(diffusion_h3_trainer)
assert dtype_rule in h3_src
assert dtype_rule in inspect.getsource(diffusion_dit_trainer)
diff --git a/studio/backend/tests/test_diffusion_xpu_training.py b/studio/backend/tests/test_diffusion_xpu_training.py
new file mode 100644
index 00000000000..f555c808ae9
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_xpu_training.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU-only unit tests for Intel XPU training on the two flow-matching trainers (#9524).
+
+These pin DECISIONS (device, dtype, which cache is cleared, what is recorded), never Intel kernels:
+CI has no Intel GPU. Keep the fake ``torch.xpu`` a bare namespace -- a MagicMock manufactures
+attributes, so every "this torch lacks that API" case would pass vacuously.
+"""
+
+from __future__ import annotations
+
+import random
+import types
+
+import pytest
+import torch
+
+import core.training.diffusion_train_common as common
+from core.training import diffusion_dit_trainer as dit
+from core.training import diffusion_h3_trainer as h3
+from core.training.diffusion_train_common import (
+ DiffusionLoraConfig,
+ effective_mixed_precision,
+ native_bf16_supported_xpu,
+ resolve_train_device,
+)
+
+
+def _fake_xpu(**attrs):
+ return types.SimpleNamespace(**attrs)
+
+
+def host(monkeypatch):
+ def _set(*, cuda: bool, xpu = None):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: cuda)
+ if xpu is None:
+ monkeypatch.delattr(torch, "xpu", raising = False)
+ else:
+ monkeypatch.setattr(torch, "xpu", xpu, raising = False)
+
+ return _set
+
+
+def test_cuda_still_wins_on_a_box_that_has_both(host):
+ host(cuda = True, xpu = _fake_xpu(is_available = lambda: True))
+ assert resolve_train_device() == "cuda"
+
+
+def test_an_xpu_box_resolves_to_xpu_not_cpu(host):
+ """#9524: the bnb quantizer already placed the transformer on xpu:0, so "cpu" here split them."""
+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True))
+ assert resolve_train_device() == "xpu"
+
+
+ "xpu",
+ [
+ None, # a torch build with no xpu module at all
+ _fake_xpu(), # present, but predates is_available()
+ _fake_xpu(is_available = True), # present, non-callable
+ _fake_xpu(is_available = lambda: False), # present, no device
+ ],
+ ids = ["absent", "no-probe", "non-callable", "unavailable"],
+)
+def test_a_host_without_a_usable_xpu_stays_on_cpu(host, xpu):
+ host(cuda = False, xpu = xpu)
+ assert resolve_train_device() == "cpu"
+
+
+def test_an_uninitialised_xpu_driver_falls_through_instead_of_killing_the_run(host):
+ """A raising probe must degrade to CPU, as dit_accelerator_missing_reason does: unguarded, an
+ uninitialised driver crashed the run."""
+
+ def _boom():
+ raise RuntimeError("driver not initialised")
+
+ host(cuda = False, xpu = _fake_xpu(is_available = _boom))
+ assert resolve_train_device() == "cpu"
+
+
+def test_an_emulation_only_xpu_is_refused(host):
+ """is_bf16_supported() defaults including_emulation=True and short-circuits, so the bare call
+ answers True for every available XPU: not a capability check."""
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),
+ ),
+ )
+ assert torch.xpu.is_bf16_supported() is True # what the bare call claims
+ assert native_bf16_supported_xpu() is False # what the hardware can actually do
+
+
+def test_a_native_bf16_xpu_is_accepted(host):
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: True,
+ ),
+ )
+ assert native_bf16_supported_xpu() is True
+
+
+def test_a_torch_predating_the_emulation_flag_still_answers(host):
+ """On a torch predating including_emulation the no-argument answer is the only one there is."""
+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True, is_bf16_supported = lambda: True))
+ assert native_bf16_supported_xpu() is True
+
+
+ "probe",
+ [None, True],
+ ids = ["absent", "non-callable"],
+)
+def test_an_xpu_without_a_usable_bf16_probe_is_refused(host, probe):
+ attrs = {"is_available": lambda: True}
+ if probe is not None:
+ attrs["is_bf16_supported"] = probe
+ host(cuda = False, xpu = _fake_xpu(**attrs))
+ assert native_bf16_supported_xpu() is False
+
+
+def _dit_cfg():
+ return DiffusionLoraConfig(
+ base_model = "black-forest-labs/FLUX.1-dev",
+ data_dir = "/tmp/d",
+ output_dir = "/tmp/o",
+ instance_prompt = "p",
+ mixed_precision = "bf16",
+ )
+
+
+def _h3_cfg():
+ return DiffusionLoraConfig(
+ base_model = "MiniMaxAI/MiniMax-H3",
+ data_dir = "/tmp/clips",
+ output_dir = "/tmp/out",
+ mixed_precision = "bf16",
+ resolution = 768,
+ )
+
+
+class _Cut(Exception):
+ pass
+
+
+def _decide(monkeypatch, module, entry, cfg):
+ """The (device, weight_dtype) the REAL entry point bound, read from its live frame.
+
+ Cut at _assert_trusted_base_model, the statement right after weight_dtype: observes the decision
+ itself rather than a copy of the rule, and nothing downstream (download, cache, training) runs.
+ """
+
+ def _cut(*_a, **_k):
+ raise _Cut()
+
+ monkeypatch.setattr(module, "_assert_trusted_base_model", _cut)
+ # manual_seed fans out into torch.xpu.manual_seed_all, which the bare fake lacks; it runs
+ # before the decision and cannot influence it.
+ monkeypatch.setattr(torch, "manual_seed", lambda *_a, **_k: None)
+
+ try:
+ entry(cfg)
+ except _Cut as exc:
+ frame = None
+ tb = exc.__traceback__
+ while tb is not None:
+ if tb.tb_frame.f_code.co_name in ("run_dit_lora_training", "run_h3_lora_training"):
+ frame = tb.tb_frame
+ tb = tb.tb_next
+ return frame.f_locals["device"], frame.f_locals["weight_dtype"]
+ raise AssertionError("the run did not reach the cut")
+
+
+ ("module", "entry", "cfg"),
+ [(dit, dit.run_dit_lora_training, _dit_cfg()), (h3, h3.run_h3_lora_training, _h3_cfg())],
+ ids = ["dit", "h3"],
+)
+def test_both_flow_trainers_run_an_xpu_box_in_bf16_on_the_xpu(
+ monkeypatch, host, module, entry, cfg
+):
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: True,
+ ),
+ )
+ assert _decide(monkeypatch, module, entry, cfg) == ("xpu", torch.bfloat16)
+
+
+ ("module", "entry", "cfg"),
+ [(dit, dit.run_dit_lora_training, _dit_cfg()), (h3, h3.run_h3_lora_training, _h3_cfg())],
+ ids = ["dit", "h3"],
+)
+def test_both_flow_trainers_refuse_an_emulation_only_xpu(monkeypatch, host, module, entry, cfg):
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),
+ ),
+ )
+ with pytest.raises(ValueError, match = "bfloat16-capable"):
+ _decide(monkeypatch, module, entry, cfg)
+
+
[email protected]("module", [dit, h3], ids = ["dit", "h3"])
+def test_every_phase_boundary_frees_the_selected_accelerator(module):
+ """Both boundaries free the text encoders and VAE right before the multi-GB transformer load,
+ so a CUDA-only clear leaves that memory resident for it on an XPU box."""
+ import inspect
+
+ src = inspect.getsource(module)
+ cuda_frees = src.count("torch.cuda.empty_cache()")
+ xpu_frees = src.count("torch.xpu.empty_cache()")
+ assert cuda_frees > 0
+ assert xpu_frees == cuda_frees
+
+
+def test_an_emulation_only_xpu_is_refused_before_the_route_evicts_anything(host):
+ """The route calls this BEFORE teardown; dit_accelerator_missing_reason accepts any available
+ XPU, so a child-only guard evicted the user's models and then failed."""
+ from core.training.diffusion_train_common import bf16_unsupported_reason
+
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),
+ ),
+ )
+ reason = bf16_unsupported_reason("krea-2")
+ assert reason and "bf16 natively" in reason
+ host(
+ cuda = False,
+ xpu = _fake_xpu(
+ is_available = lambda: True,
+ is_bf16_supported = lambda including_emulation = True: True,
+ ),
+ )
+ assert bf16_unsupported_reason("krea-2") is None
+
+
+def test_an_unprobeable_xpu_is_left_to_the_child_not_refused_up_front(host):
+ """The preflight runs before teardown on a host whose XPU may not be interrogable at all, so an
+ undeterminable capability must fail OPEN. Collapsing it to "unsupported" refused nf4 on every
+ such host (caught by test_training_precision_preflight_error)."""
+ from core.training.diffusion_train_common import (
+ bf16_unsupported_reason,
+ native_bf16_supported_xpu,
+ xpu_native_bf16_probe,
+ )
+
+ def _unprobeable(including_emulation = True):
+ raise RuntimeError("no device properties")
+
+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True, is_bf16_supported = _unprobeable))
+ assert xpu_native_bf16_probe() is None
+ assert bf16_unsupported_reason("krea-2") is None # route proceeds
+ assert native_bf16_supported_xpu() is False # child still refuses, on the device itself
+
+
+def test_an_xpu_run_captures_and_restores_its_own_noise_generator(host, monkeypatch):
+ """The loops draw randn_like on the training device, so an XPU run's per-step noise lives in the
+ XPU generator; capturing only torch_cpu resumed it from a fresh seed."""
+ from core.training.diffusion_checkpoint import capture_rng_state, restore_rng_state
+
+ seen = {"set": []}
+ states = [torch.tensor([7, 7, 7], dtype = torch.uint8)]
+ fake = _fake_xpu(
+ is_available = lambda: True,
+ device_count = lambda: 1,
+ get_rng_state_all = lambda: states,
+ set_rng_state = lambda st, i: seen["set"].append((i, st.tolist())),
+ )
+ host(cuda = False, xpu = fake)
+
+ captured = capture_rng_state()
+ assert "torch_xpu_0" in captured["tensors"]
+
+ restore_rng_state(captured["json"], captured["tensors"])
+ assert seen["set"] == [(0, [7, 7, 7])]
+
+
+def test_a_checkpoint_cannot_silently_resume_across_accelerator_backends(host, tmp_path):
+ """Recording bf16 for XPU too (above) removed the accidental barrier that used to stop a
+ CUDA checkpoint resuming on XPU: the precisions now match, the identity carries no backend, and
+ the restore finds no torch_xpu_* key, so the device generator stays freshly seeded while the
+ resume reports success. Refused at the preflight, before teardown."""
+ from pathlib import Path
+
+ from core.training.diffusion_checkpoint import (
+ ResumeError,
+ _assert_required_state,
+ capture_rng_state,
+ )
+
+ complete = {
+ "kind": "image",
+ "sampler": {"pos": 0},
+ "files": {k: f"{k}.pt" for k in ("adapter", "optimizer", "scheduler", "rng")},
+ }
+ written_here = capture_rng_state({"loop": random.Random(1), "variant": random.Random(2)})["json"]
+ assert written_here["accelerator"] in ("cuda", "xpu", "cpu")
+
+ # Same host: accepted.
+ _assert_required_state(Path("ckpt"), {**complete, "rng": written_here})
+
+ # Written on the other backend: refused, and the message says which.
+ other = "xpu" if written_here["accelerator"] != "xpu" else "cuda"
+ with pytest.raises(ResumeError, match = "this accelerator"):
+ _assert_required_state(
+ Path("ckpt"), {**complete, "rng": {**written_here, "accelerator": other}}
+ )
+
+ # A bundle predating the field must still resume: unknown is not a mismatch.
+ legacy = {k: v for k, v in written_here.items() if k != "accelerator"}
+ _assert_required_state(Path("ckpt"), {**complete, "rng": legacy})
+
+
+def test_an_xpu_flow_run_records_the_bf16_it_actually_trains_in(host):
+ """identity_for_config stores this and a resume is refused when it disagrees, so a CUDA-only
+ rule would record "no" for a run the loop trained in bf16."""
+ cfg = _dit_cfg().normalized()
+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True))
+ assert effective_mixed_precision(cfg) == "bf16"
+ host(cuda = False, xpu = None)
+ assert effective_mixed_precision(cfg) == "no"
+ assert common.resolve_train_device() == "cpu"