Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fixes to XPU offloading during training (#9534)
Joseph Antonuccifac17ac642f5
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.pyindex d025b9c63fd..83c4f880734 100644--- a/studio/backend/core/training/diffusion_checkpoint.py+++ b/studio/backend/core/training/diffusion_checkpoint.py@@ -590,6 +590,9 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any"python": _random_state_to_json(random.getstate()),"streams": {},"numpy": None,+ # Which accelerator the torch_* device tensors below belong to. The resume preflight reads it: the device+ # generator of one backend says nothing on another, and nothing else in the identity distinguishes them.+ "accelerator": _rng_accelerator(),}for name, stream in (streams or {}).items():try:@@ -624,6 +627,14 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any# (which only requires torch_cpu) offered it and the restore then left the CUDA generator freshly# seeded, silently changing every latent and timestep draw.tensors = {}+ elif _xpu_available():+ # The flow trainers draw randn_like on the training device, so an XPU run's noise stream lives in the XPU+ # generator; capturing only torch_cpu resumed it from a fresh seed. Same all-or-nothing rule as CUDA.+ try:+ for i, state in enumerate(torch.xpu.get_rng_state_all()):+ tensors[f"torch_xpu_{i}"] = state+ except Exception: # noqa: BLE001 -- one device erroring loses the whole capture+ tensors = {}except Exception: # noqa: BLE001 -- torch RNG capture is best-efforttensors = {}return {"json": payload, "tensors": tensors}@@ -685,10 +696,39 @@ def restore_rng_state(if state is None:continuetorch.cuda.set_rng_state(state.cpu().to(torch.uint8), i)+ elif _xpu_available():+ # Per device for the same reason as CUDA above: set_rng_state_all needs one state per visible device.+ for i in range(torch.xpu.device_count()):+ state = tensors.get(f"torch_xpu_{i}")+ if state is None:+ continue+ torch.xpu.set_rng_state(state.cpu().to(torch.uint8), i)except Exception: # noqa: BLE001 -- best-effort restore, never fatalpass+def _rng_accelerator() -> str:+ """The backend whose device generator a capture holds: ``cuda``, ``xpu`` or ``cpu``."""+ try:+ import torch+ if torch.cuda.is_available():+ return "cuda"+ except Exception: # noqa: BLE001 -- probe failure -> no CUDA generator captured+ pass+ return "xpu" if _xpu_available() else "cpu"+++def _xpu_available() -> bool:+ """Guarded like ``resolve_train_device``'s probe: an uninitialised driver must not turn a+ best-effort RNG capture into a raise."""+ try:+ import torch+ fn = getattr(getattr(torch, "xpu", None), "is_available", None)+ return bool(fn()) if callable(fn) else False+ except Exception: # noqa: BLE001 -- probe failure -> no XPU generator to capture+ return False++def _random_state_to_json(state: Any) -> Optional[list[Any]]:"""``random.Random.getstate()`` is ``(version, tuple[int, ...], gauss)``; JSON has no tuples, sostore it as nested lists."""@@ -1718,6 +1758,16 @@ def _assert_required_state(path: Path, manifest: dict[str, Any]) -> None:isinstance(saved_streams.get(name), (list, tuple)) for name in _TRAINER_RNG_STREAMS):missing.append("the trainer's random-number streams")+ # Same failure one level down: a CUDA bundle carries only torch_cuda_* keys, so resuming it on XPU (or the+ # reverse) leaves the destination generator freshly seeded and reports a clean resume. Nothing else catches it,+ # since the identity records the effective precision and that is bf16 on both. Only a KNOWN mismatch counts: a+ # bundle predating this field must still resume.+ saved_accel = rng_manifest.get("accelerator") if isinstance(rng_manifest, dict) else None+ if isinstance(saved_accel, str) and saved_accel and saved_accel != _rng_accelerator():+ missing.append(+ f"the random-number state for this accelerator (written on {saved_accel}, "+ f"resuming on {_rng_accelerator()})"+ )if not missing:returnraise ResumeError(diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 7cb73740679..f6c349c6da3 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -57,8 +57,10 @@ from core.training.diffusion_train_common import (discover_image_caption_pairs,has_functional_torchao,native_bf16_supported,+ native_bf16_supported_xpu,PermutationBatchSampler,repo_is_prequantized,+ resolve_train_device,resolve_train_steps,restore_resume_state,write_resume_checkpoint,@@ -268,13 +270,14 @@ def _bnb_4bit_config():_repo_is_prequantized = repo_is_prequantized-def _load_quantized_transformer(transformer_cls, cfg):+def _load_quantized_transformer(transformer_cls, cfg, device):"""Load ``cfg.base_model``'s transformer subfolder as a trainable nf4 QLoRA module."""import torchreturn transformer_cls.from_pretrained(cfg.base_model,subfolder = "transformer",quantization_config = _bnb_4bit_config(),+ device_map = {"": device},torch_dtype = torch.bfloat16,token = cfg.hf_token,)@@ -307,7 +310,7 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):if base_precision == "nf4":if not repo_is_prequantized(cfg.base_model):- return _load_quantized_transformer(transformer_cls, cfg)+ return _load_quantized_transformer(transformer_cls, cfg, device)transformer = transformer_cls.from_pretrained(cfg.base_model,subfolder = "transformer",@@ -1853,15 +1856,20 @@ def run_dit_lora_training(save_on_stop = Falsereturn True- device = "cuda" if torch.cuda.is_available() else "cpu"- # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since is_bf16_supported() counts- # emulation.+ device = resolve_train_device()+ # bf16 throughout (fp32 on a CPU-only box, to keep import/unit tests architecture-agnostic). Both accelerator+ # guards gate on NATIVE bf16, since is_bf16_supported() counts emulation on CUDA and on XPU alike.if device == "cuda" and not native_bf16_supported():raise ValueError("This trainer requires a bfloat16-capable GPU (Ampere or newer); ""this CUDA device does not support bf16.")- weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32+ if device == "xpu" and not native_bf16_supported_xpu():+ raise ValueError(+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "+ "support bf16 natively."+ )+ weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32_assert_trusted_base_model(cfg.base_model)# Check the repo this run will FETCH: the canonical id would raise for a gated base already redirected to its@@ -2002,6 +2010,8 @@ def _train_dit(gc.collect()if device == "cuda":torch.cuda.empty_cache()+ elif device == "xpu":+ torch.xpu.empty_cache()# The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.if use_cache:@@ -2043,6 +2053,8 @@ def _train_dit(gc.collect()if device == "cuda":torch.cuda.empty_cache()+ elif device == "xpu":+ torch.xpu.empty_cache()# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.variant_rng = random.Random(cfg.seed + 1)@@ -2209,8 +2221,8 @@ def _train_dit(# bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles the fp32 LoRA# params with the bnb 4-bit base matmuls.autocast = (- torch.autocast(device_type = "cuda", dtype = torch.bfloat16)- if device == "cuda"+ torch.autocast(device_type = device, dtype = torch.bfloat16)+ if device in ("cuda", "xpu")else nullcontext())for opt_step in range(resumed, cfg.train_steps):diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.pyindex 1fa3360eb93..ac266ae0dde 100644--- a/studio/backend/core/training/diffusion_h3_trainer.py+++ b/studio/backend/core/training/diffusion_h3_trainer.py@@ -83,6 +83,8 @@ from core.training.diffusion_train_common import (_restore_perf_flags,h3_train_unsupported_reason,native_bf16_supported,+ native_bf16_supported_xpu,+ resolve_train_device,resolve_train_steps,train_recipe_overrides,)@@ -313,6 +315,7 @@ def _load_transformer(cfg, device, base_precision):cfg.base_model,subfolder = "transformer",quantization_config = quant,+ device_map = {"": device},torch_dtype = torch.bfloat16,token = cfg.hf_token,cache_dir = cache_dir,@@ -479,13 +482,18 @@ def run_h3_lora_training(save_on_stop = Falsereturn True- device = "cuda" if torch.cuda.is_available() else "cpu"+ device = resolve_train_device()if device == "cuda" and not native_bf16_supported():raise ValueError("This trainer requires a bfloat16-capable GPU (Ampere or newer); ""this CUDA device does not support bf16.")- weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32+ if device == "xpu" and not native_bf16_supported_xpu():+ raise ValueError(+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "+ "support bf16 natively."+ )+ weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32# allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local MiniMax-H3# pipeline carries modular_model_index.json and no model_index.json, so the conventional shape@@ -544,6 +552,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savgc.collect()if device == "cuda":torch.cuda.empty_cache()+ elif device == "xpu":+ torch.xpu.empty_cache()# Phase 2: the clip cache. One canvas for the run, from the FIRST clip's aspect ratio: every other clip is# cover-cropped onto it, so a mixed-aspect dataset trains on one geometry.@@ -598,6 +608,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savgc.collect()if device == "cuda":torch.cuda.empty_cache()+ elif device == "xpu":+ torch.xpu.empty_cache()# Phase 3: the denoiser.base_precision = cfg.base_precision if cfg.base_precision != "auto" else "nf4"@@ -670,8 +682,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savpatch = tuple(transformer.config.patch_size)index_sampler = PermutationBatchSampler(len(clip_paths), rng)autocast = (- torch.autocast(device_type = "cuda", dtype = torch.bfloat16)- if device == "cuda"+ torch.autocast(device_type = device, dtype = torch.bfloat16)+ if device in ("cuda", "xpu")else nullcontext())stopped = Falsediff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex c9316c24d59..372fe49fc19 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -613,10 +613,10 @@ def effective_mixed_precision(cfg: Any) -> str:requested = str(getattr(cfg, "mixed_precision", "") or "")if str(getattr(cfg, "resolved_family", "") or "").strip().lower() in _FLOW_TRAIN_FAMILIES:- # No flow-matching trainer reads mixed_precision (weight_dtype is bf16 on CUDA, fp32 otherwise), so recording- # the REQUEST failed a later bf16 resume as a precision mismatch between identical runs. Keyed on- # _FLOW_TRAIN_FAMILIES so the answer follows the weight dtype.- return "bf16" if torch.cuda.is_available() else "no"+ # No flow trainer reads mixed_precision (weight_dtype is bf16 on an accelerator, fp32 otherwise), so recording+ # the REQUEST failed a later bf16 resume as a mismatch between identical runs. Resolve the device the way the+ # loops do, or an XPU run trains in bf16 and records "no".+ return "bf16" if resolve_train_device() in ("cuda", "xpu") else "no"if not torch.cuda.is_available():return "no"if requested == "bf16" and not native_bf16_supported():@@ -644,6 +644,62 @@ def native_bf16_supported() -> bool:return False+def resolve_train_device() -> str:+ """The device a flow-matching trainer runs on: CUDA first (so a box with both is unaffected),+ then Intel XPU, else CPU.++ Probes are guarded individually like ``dit_accelerator_missing_reason``: an uninitialised driver+ whose ``is_available()`` raises must fall through, not kill the run. Shared so the two trainers+ and the precision the run RECORDS cannot disagree."""+ import torch # noqa: PLC0415 -- keep the import list light for the training subprocess++ def _probe(module) -> bool:+ try:+ fn = getattr(module, "is_available", None)+ return bool(fn()) if callable(fn) else False+ except Exception: # noqa: BLE001 -- probe failure -> this accelerator is not usable+ return False++ if _probe(getattr(torch, "cuda", None)):+ return "cuda"+ if _probe(getattr(torch, "xpu", None)):+ return "xpu"+ return "cpu"+++def xpu_native_bf16_probe() -> Optional[bool]:+ """Native-bf16 on the live XPU, or None when the capability cannot be determined. Never raises.++ ``is_bf16_supported()`` defaults to ``including_emulation=True`` and short-circuits before+ reading ``has_bfloat16_conversions``, so the bare call answers True for EVERY available XPU:+ the same emulation trap ``native_bf16_supported`` avoids on the CUDA side. Ask explicitly.++ Tri-state because ``get_device_properties()`` raises when no device is really there: collapsing+ that to False made the pre-eviction preflight refuse nf4 on any host whose XPU cannot be+ interrogated, a probe failing CLOSED."""+ import torch # noqa: PLC0415++ fn = getattr(getattr(torch, "xpu", None), "is_bf16_supported", None)+ if not callable(fn):+ return None+ try:+ return bool(fn(including_emulation = False))+ except TypeError:+ # A torch predating the including_emulation parameter: its answer is the only one there is.+ try:+ return bool(fn())+ except Exception: # noqa: BLE001 -- unprobeable+ return None+ except Exception: # noqa: BLE001 -- unprobeable+ return None+++def native_bf16_supported_xpu() -> bool:+ """``xpu_native_bf16_probe`` collapsed for the trainer child, which runs ON the device: there an+ unprobeable capability is a refusal, matching ``native_bf16_supported`` on the CUDA side."""+ return xpu_native_bf16_probe() is True++def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:"""Return a user-facing error string if ``resolved_family`` needs bf16 compute the live GPUcannot provide, else None. The DiT trainer requires Ampere or newer and otherwise raises deep@@ -661,6 +717,16 @@ def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:"This trainer requires a bfloat16-capable GPU (Ampere or newer); this CUDA ""device does not support bf16. Train the DiT families on a newer GPU.")+ # dit_accelerator_missing_reason accepts any available XPU, so without this the route+ # admitted an emulation-only XPU, evicted the resident models, and only then hit the+ # trainer's own guard: the eviction ordering this function exists to protect. Only a+ # DEFINITE no rejects here; an unprobeable XPU is left to the child, since a preflight that+ # fails closed would refuse hosts that train fine.+ if resolve_train_device() == "xpu" and xpu_native_bf16_probe() is False:+ return (+ "This trainer requires a bfloat16-capable GPU; this XPU device does not "+ "support bf16 natively. Train the DiT families on a newer GPU."+ )except Exception: # noqa: BLE001 -- torch probe failure must not block a startreturn Nonereturn Nonediff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.pyindex 0f4a9fe4be2..56c0a6ab167 100644--- a/studio/backend/tests/test_diffusion_h3_trainer.py+++ b/studio/backend/tests/test_diffusion_h3_trainer.py@@ -1273,7 +1273,7 @@ def test_the_precision_recorded_for_h3_is_the_one_its_loop_runs_in():# The claim the equality rests on: the same weight-dtype rule in both loops, and no reader of# mixed_precision in the H3 one.- dtype_rule = 'weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32'+ dtype_rule = 'weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32'h3_src = inspect.getsource(diffusion_h3_trainer)assert dtype_rule in h3_srcassert dtype_rule in inspect.getsource(diffusion_dit_trainer)diff --git a/studio/backend/tests/test_diffusion_xpu_training.py b/studio/backend/tests/test_diffusion_xpu_training.pynew file mode 100644index 00000000000..f555c808ae9--- /dev/null+++ b/studio/backend/tests/test_diffusion_xpu_training.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU-only unit tests for Intel XPU training on the two flow-matching trainers (#9524).++These pin DECISIONS (device, dtype, which cache is cleared, what is recorded), never Intel kernels:+CI has no Intel GPU. Keep the fake ``torch.xpu`` a bare namespace -- a MagicMock manufactures+attributes, so every "this torch lacks that API" case would pass vacuously.+"""++from __future__ import annotations++import random+import types++import pytest+import torch++import core.training.diffusion_train_common as common+from core.training import diffusion_dit_trainer as dit+from core.training import diffusion_h3_trainer as h3+from core.training.diffusion_train_common import (+ DiffusionLoraConfig,+ effective_mixed_precision,+ native_bf16_supported_xpu,+ resolve_train_device,+)+++def _fake_xpu(**attrs):+ return types.SimpleNamespace(**attrs)+++def host(monkeypatch):+ def _set(*, cuda: bool, xpu = None):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: cuda)+ if xpu is None:+ monkeypatch.delattr(torch, "xpu", raising = False)+ else:+ monkeypatch.setattr(torch, "xpu", xpu, raising = False)++ return _set+++def test_cuda_still_wins_on_a_box_that_has_both(host):+ host(cuda = True, xpu = _fake_xpu(is_available = lambda: True))+ assert resolve_train_device() == "cuda"+++def test_an_xpu_box_resolves_to_xpu_not_cpu(host):+ """#9524: the bnb quantizer already placed the transformer on xpu:0, so "cpu" here split them."""+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True))+ assert resolve_train_device() == "xpu"+++ "xpu",+ [+ None, # a torch build with no xpu module at all+ _fake_xpu(), # present, but predates is_available()+ _fake_xpu(is_available = True), # present, non-callable+ _fake_xpu(is_available = lambda: False), # present, no device+ ],+ ids = ["absent", "no-probe", "non-callable", "unavailable"],+)+def test_a_host_without_a_usable_xpu_stays_on_cpu(host, xpu):+ host(cuda = False, xpu = xpu)+ assert resolve_train_device() == "cpu"+++def test_an_uninitialised_xpu_driver_falls_through_instead_of_killing_the_run(host):+ """A raising probe must degrade to CPU, as dit_accelerator_missing_reason does: unguarded, an+ uninitialised driver crashed the run."""++ def _boom():+ raise RuntimeError("driver not initialised")++ host(cuda = False, xpu = _fake_xpu(is_available = _boom))+ assert resolve_train_device() == "cpu"+++def test_an_emulation_only_xpu_is_refused(host):+ """is_bf16_supported() defaults including_emulation=True and short-circuits, so the bare call+ answers True for every available XPU: not a capability check."""+ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),+ ),+ )+ assert torch.xpu.is_bf16_supported() is True # what the bare call claims+ assert native_bf16_supported_xpu() is False # what the hardware can actually do+++def test_a_native_bf16_xpu_is_accepted(host):+ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: True,+ ),+ )+ assert native_bf16_supported_xpu() is True+++def test_a_torch_predating_the_emulation_flag_still_answers(host):+ """On a torch predating including_emulation the no-argument answer is the only one there is."""+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True, is_bf16_supported = lambda: True))+ assert native_bf16_supported_xpu() is True+++ "probe",+ [None, True],+ ids = ["absent", "non-callable"],+)+def test_an_xpu_without_a_usable_bf16_probe_is_refused(host, probe):+ attrs = {"is_available": lambda: True}+ if probe is not None:+ attrs["is_bf16_supported"] = probe+ host(cuda = False, xpu = _fake_xpu(**attrs))+ assert native_bf16_supported_xpu() is False+++def _dit_cfg():+ return DiffusionLoraConfig(+ base_model = "black-forest-labs/FLUX.1-dev",+ data_dir = "/tmp/d",+ output_dir = "/tmp/o",+ instance_prompt = "p",+ mixed_precision = "bf16",+ )+++def _h3_cfg():+ return DiffusionLoraConfig(+ base_model = "MiniMaxAI/MiniMax-H3",+ data_dir = "/tmp/clips",+ output_dir = "/tmp/out",+ mixed_precision = "bf16",+ resolution = 768,+ )+++class _Cut(Exception):+ pass+++def _decide(monkeypatch, module, entry, cfg):+ """The (device, weight_dtype) the REAL entry point bound, read from its live frame.++ Cut at _assert_trusted_base_model, the statement right after weight_dtype: observes the decision+ itself rather than a copy of the rule, and nothing downstream (download, cache, training) runs.+ """++ def _cut(*_a, **_k):+ raise _Cut()++ monkeypatch.setattr(module, "_assert_trusted_base_model", _cut)+ # manual_seed fans out into torch.xpu.manual_seed_all, which the bare fake lacks; it runs+ # before the decision and cannot influence it.+ monkeypatch.setattr(torch, "manual_seed", lambda *_a, **_k: None)++ try:+ entry(cfg)+ except _Cut as exc:+ frame = None+ tb = exc.__traceback__+ while tb is not None:+ if tb.tb_frame.f_code.co_name in ("run_dit_lora_training", "run_h3_lora_training"):+ frame = tb.tb_frame+ tb = tb.tb_next+ return frame.f_locals["device"], frame.f_locals["weight_dtype"]+ raise AssertionError("the run did not reach the cut")+++ ("module", "entry", "cfg"),+ [(dit, dit.run_dit_lora_training, _dit_cfg()), (h3, h3.run_h3_lora_training, _h3_cfg())],+ ids = ["dit", "h3"],+)+def test_both_flow_trainers_run_an_xpu_box_in_bf16_on_the_xpu(+ monkeypatch, host, module, entry, cfg+):+ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: True,+ ),+ )+ assert _decide(monkeypatch, module, entry, cfg) == ("xpu", torch.bfloat16)+++ ("module", "entry", "cfg"),+ [(dit, dit.run_dit_lora_training, _dit_cfg()), (h3, h3.run_h3_lora_training, _h3_cfg())],+ ids = ["dit", "h3"],+)+def test_both_flow_trainers_refuse_an_emulation_only_xpu(monkeypatch, host, module, entry, cfg):+ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),+ ),+ )+ with pytest.raises(ValueError, match = "bfloat16-capable"):+ _decide(monkeypatch, module, entry, cfg)++[email protected]("module", [dit, h3], ids = ["dit", "h3"])+def test_every_phase_boundary_frees_the_selected_accelerator(module):+ """Both boundaries free the text encoders and VAE right before the multi-GB transformer load,+ so a CUDA-only clear leaves that memory resident for it on an XPU box."""+ import inspect++ src = inspect.getsource(module)+ cuda_frees = src.count("torch.cuda.empty_cache()")+ xpu_frees = src.count("torch.xpu.empty_cache()")+ assert cuda_frees > 0+ assert xpu_frees == cuda_frees+++def test_an_emulation_only_xpu_is_refused_before_the_route_evicts_anything(host):+ """The route calls this BEFORE teardown; dit_accelerator_missing_reason accepts any available+ XPU, so a child-only guard evicted the user's models and then failed."""+ from core.training.diffusion_train_common import bf16_unsupported_reason++ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: bool(including_emulation),+ ),+ )+ reason = bf16_unsupported_reason("krea-2")+ assert reason and "bf16 natively" in reason+ host(+ cuda = False,+ xpu = _fake_xpu(+ is_available = lambda: True,+ is_bf16_supported = lambda including_emulation = True: True,+ ),+ )+ assert bf16_unsupported_reason("krea-2") is None+++def test_an_unprobeable_xpu_is_left_to_the_child_not_refused_up_front(host):+ """The preflight runs before teardown on a host whose XPU may not be interrogable at all, so an+ undeterminable capability must fail OPEN. Collapsing it to "unsupported" refused nf4 on every+ such host (caught by test_training_precision_preflight_error)."""+ from core.training.diffusion_train_common import (+ bf16_unsupported_reason,+ native_bf16_supported_xpu,+ xpu_native_bf16_probe,+ )++ def _unprobeable(including_emulation = True):+ raise RuntimeError("no device properties")++ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True, is_bf16_supported = _unprobeable))+ assert xpu_native_bf16_probe() is None+ assert bf16_unsupported_reason("krea-2") is None # route proceeds+ assert native_bf16_supported_xpu() is False # child still refuses, on the device itself+++def test_an_xpu_run_captures_and_restores_its_own_noise_generator(host, monkeypatch):+ """The loops draw randn_like on the training device, so an XPU run's per-step noise lives in the+ XPU generator; capturing only torch_cpu resumed it from a fresh seed."""+ from core.training.diffusion_checkpoint import capture_rng_state, restore_rng_state++ seen = {"set": []}+ states = [torch.tensor([7, 7, 7], dtype = torch.uint8)]+ fake = _fake_xpu(+ is_available = lambda: True,+ device_count = lambda: 1,+ get_rng_state_all = lambda: states,+ set_rng_state = lambda st, i: seen["set"].append((i, st.tolist())),+ )+ host(cuda = False, xpu = fake)++ captured = capture_rng_state()+ assert "torch_xpu_0" in captured["tensors"]++ restore_rng_state(captured["json"], captured["tensors"])+ assert seen["set"] == [(0, [7, 7, 7])]+++def test_a_checkpoint_cannot_silently_resume_across_accelerator_backends(host, tmp_path):+ """Recording bf16 for XPU too (above) removed the accidental barrier that used to stop a+ CUDA checkpoint resuming on XPU: the precisions now match, the identity carries no backend, and+ the restore finds no torch_xpu_* key, so the device generator stays freshly seeded while the+ resume reports success. Refused at the preflight, before teardown."""+ from pathlib import Path++ from core.training.diffusion_checkpoint import (+ ResumeError,+ _assert_required_state,+ capture_rng_state,+ )++ complete = {+ "kind": "image",+ "sampler": {"pos": 0},+ "files": {k: f"{k}.pt" for k in ("adapter", "optimizer", "scheduler", "rng")},+ }+ written_here = capture_rng_state({"loop": random.Random(1), "variant": random.Random(2)})["json"]+ assert written_here["accelerator"] in ("cuda", "xpu", "cpu")++ # Same host: accepted.+ _assert_required_state(Path("ckpt"), {**complete, "rng": written_here})++ # Written on the other backend: refused, and the message says which.+ other = "xpu" if written_here["accelerator"] != "xpu" else "cuda"+ with pytest.raises(ResumeError, match = "this accelerator"):+ _assert_required_state(+ Path("ckpt"), {**complete, "rng": {**written_here, "accelerator": other}}+ )++ # A bundle predating the field must still resume: unknown is not a mismatch.+ legacy = {k: v for k, v in written_here.items() if k != "accelerator"}+ _assert_required_state(Path("ckpt"), {**complete, "rng": legacy})+++def test_an_xpu_flow_run_records_the_bf16_it_actually_trains_in(host):+ """identity_for_config stores this and a resume is refused when it disagrees, so a CUDA-only+ rule would record "no" for a run the loop trained in bf16."""+ cfg = _dit_cfg().normalized()+ host(cuda = False, xpu = _fake_xpu(is_available = lambda: True))+ assert effective_mixed_precision(cfg) == "bf16"+ host(cuda = False, xpu = None)+ assert effective_mixed_precision(cfg) == "no"+ assert common.resolve_train_device() == "cpu"