Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
float8_training: keep Accelerate dispatch hooks on converted layers (#15363)
Daniel Han8f390e490ff5
diff --git a/tests/test_float8_training.py b/tests/test_float8_training.pyindex a83824664c1..afcdc2ff5e7 100644--- a/tests/test_float8_training.py+++ b/tests/test_float8_training.py@@ -407,3 +407,58 @@ def test_probe_scales_follow_each_cast_config(monkeypatch):(torch.float8_e5m2, e4m3, (), ()),]assert _probe_calls(monkeypatch, "rowwise") == [(e4m3, e4m3, (16, 1), (1, 16))]+++def _counting_align_hook():+ from accelerate.hooks import AlignDevicesHook+ class CountingAlignDevicesHook(AlignDevicesHook):+ calls = []++ def pre_forward(self, module, *args, **kwargs):+ CountingAlignDevicesHook.calls.append(id(module))+ return super().pre_forward(module, *args, **kwargs)++ return CountingAlignDevicesHook+++def test_accelerate_dispatch_hooks_survive_conversion():+ pytest.importorskip("accelerate")+ from accelerate.hooks import add_hook_to_module++ hook_cls = _counting_align_hook()+ model = _tiny_qwen3()+ hooks = {}+ for name, m in model.named_modules():+ if isinstance(m, torch.nn.Linear):+ hooks[name] = hook_cls(execution_device = torch.device("cpu"), io_same_device = True)+ add_hook_to_module(m, hooks[name])+ weights = {name: m.weight for name, m in model.named_modules() if name in hooks}++ converted = f8t.convert_model_to_float8_training(+ model, _emulated_config("tensorwise"), size_filter = False, check_hardware = False+ )+ assert converted+ modules = dict(model.named_modules())+ after = {name: m for name, m in modules.items() if getattr(m, "_hf_hook", None) is not None}+ assert set(after) == set(hooks)+ for name in converted:+ assert isinstance(modules[name], _float8_linear())+ assert modules[name]._hf_hook is hooks[name]+ assert modules[name].weight is weights[name]+ assert "init_hook" not in vars(hooks[name])++ hook_cls.calls.clear()+ ids = torch.randint(0, 256, (2, 32), generator = torch.Generator().manual_seed(1))+ out = model(input_ids = ids, labels = ids)+ out.loss.backward()+ assert torch.isfinite(out.loss)+ assert {id(modules[name]) for name in converted} <= set(hook_cls.calls)+++def test_conversion_adds_no_hooks_to_an_undispatched_model():+ model = _tiny_qwen3()+ converted = f8t.convert_model_to_float8_training(+ model, _emulated_config("tensorwise"), size_filter = False, check_hardware = False+ )+ assert converted+ assert not any(hasattr(m, "_hf_hook") for m in model.modules())diff --git a/unsloth/models/float8_training.py b/unsloth/models/float8_training.pyindex 38fcf9c65d3..bdfc276c95a 100644--- a/unsloth/models/float8_training.py+++ b/unsloth/models/float8_training.py@@ -235,6 +235,22 @@ def _check_hardware(device, recipe):) from e+def _reattach_dispatch_hooks(model, names, dispatch_hooks):+ from accelerate.hooks import add_hook_to_module+ modules = dict(model.named_modules())+ for name in names:+ hook = dispatch_hooks.get(name)+ module = modules.get(name)+ if hook is None or module is None or getattr(module, "_hf_hook", None) is not None:+ continue+ # Skip init_hook: re-placing the reused parameters would re-create them and untie shared ones.+ hook.init_hook = lambda module: module+ try:+ add_hook_to_module(module, hook)+ finally:+ del hook.init_hook++def convert_model_to_float8_training(model,float8_training = True,@@ -275,8 +291,16 @@ def convert_model_to_float8_training(devices = _selected_devices(model, module_filter_fn)for device in devices or [next((p.device for p in model.parameters()), None)]:_check_hardware(device, recipe)+ # torchao's new Float8Linear drops Accelerate dispatch hooks (device_map / offload); re-attach them.+ dispatch_hooks = {+ name: m._hf_hook+ for name, m in model.named_modules()+ if isinstance(m, torch.nn.Linear) and getattr(m, "_hf_hook", None) is not None+ }convert_to_float8_training(model, module_filter_fn = module_filter_fn, config = config)converted = [name for name, m in model.named_modules() if isinstance(m, Float8Linear)]+ if dispatch_hooks:+ _reattach_dispatch_hooks(model, converted, dispatch_hooks)if not converted:import warningswarnings.warn(