Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Preserve split-device compressed-tensors input transfers
Daniel Hanbbee0dc04fb3
diff --git a/tests/test_compressed_tensors_lazy_decompress_offload.py b/tests/test_compressed_tensors_lazy_decompress_offload.pyindex e91ee7c8a19..fd29a737d7d 100644--- a/tests/test_compressed_tensors_lazy_decompress_offload.py+++ b/tests/test_compressed_tensors_lazy_decompress_offload.py@@ -14,12 +14,8 @@# You should have received a copy of the GNU Affero General Public License# along with this program. If not, see <https://www.gnu.org/licenses/>.-"""A compressed-tensors checkpoint loaded with plain transformers in a process where Unsloth is-imported (say, reloading what Unsloth saved) is decompressed in compressed-tensors' first-forward-hook, which wraps each module's parameters in an OffloadCache. Unsloth's compiled forwards cannot-trace that cache, so Unsloth unwraps the same-device ones right after the hook runs."""-import types+from dataclasses import dataclassimport pytestimport torch@@ -48,9 +44,6 @@ class _Tiny(nn.Module):class _LazyCompressor:- """compressed-tensors' own add_decompress_hook / remove_decompression_hook, with a- decompress_model that does what 0.19's does to the module tree (from_accelerate dispatch)."""-add_decompress_hook = ModelCompressor.add_decompress_hookremove_decompression_hook = ModelCompressor.remove_decompression_hook@@ -86,7 +79,7 @@ def test_unwrap_hook_runs_after_the_decompress_hook_and_removes_itself():reference.load_state_dict(model.state_dict())x = torch.randn(2, 4)assert mcl._unwrap_offload_after_lazy_decompress(model) is True- assert mcl._unwrap_offload_after_lazy_decompress(model) is False # idempotent+ assert mcl._unwrap_offload_after_lazy_decompress(model) is Falseout = model(x)assert _caches(model) == 0assert not hasattr(model, "ct_decompress_hook")@@ -114,6 +107,211 @@ def test_real_offload_is_left_alone():assert _caches(model) == 2+def _split_device_metadata(model):+ index = 0+ for module in model.modules():+ if module._parameters or module._buffers:+ offload_module_lib.offload_module(module, torch.device("cpu"), torch.device("cpu"))+ for cache in (module._parameters, module._buffers):+ cache.onload_device = torch.device(f"cuda:{index % 2}")+ cache.offload_device = cache.onload_device+ index += 1+ return model+++def test_split_device_model_keeps_the_input_transfer():+ model = _split_device_metadata(_Tiny())+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ assert _remove_same_device_compressed_tensors_offload(model) == 2+ assert _caches(model) == 0+ assert not hasattr(model.proj, "_original_forward_func")+ assert not model.proj._forward_pre_hooks+ forward = model.proj.forward.__func__+ assert _remove_same_device_compressed_tensors_offload(model) == 0+ assert model.proj.forward.__func__ is forward+++def test_split_cleanup_preserves_encoder_outputs():+ from transformers import BartConfig, BartForConditionalGeneration+ from transformers.modeling_outputs import BaseModelOutput+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ config = BartConfig(+ vocab_size = 32,+ d_model = 8,+ encoder_layers = 1,+ decoder_layers = 1,+ encoder_attention_heads = 2,+ decoder_attention_heads = 2,+ encoder_ffn_dim = 16,+ decoder_ffn_dim = 16,+ max_position_embeddings = 8,+ dropout = 0.0,+ attention_dropout = 0.0,+ activation_dropout = 0.0,+ )+ model = BartForConditionalGeneration(config).eval()+ hidden = torch.randn(2, 3, 8, requires_grad = True)+ encoded = BaseModelOutput(last_hidden_state = hidden)+ ids = torch.tensor([[2, 3], [2, 4]])+ expected = model(encoder_outputs = encoded, decoder_input_ids = ids, use_cache = False).logits+ _split_device_metadata(model)+ assert _remove_same_device_compressed_tensors_offload(model) > 1+ actual = model(encoder_outputs = encoded, decoder_input_ids = ids, use_cache = False).logits+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)+ actual.square().sum().backward()+ assert hidden.grad is not None++[email protected]("dynamic", [False, True])+def test_split_model_output_fullgraph(dynamic):+ from transformers.modeling_outputs import BaseModelOutput+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ class Reader(nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.eye(4))++ def forward(self, encoded):+ return encoded.last_hidden_state @ self.weight++ model = _split_device_metadata(nn.Sequential(Reader(), nn.Linear(4, 4)))+ assert _remove_same_device_compressed_tensors_offload(model) == 2+ value = torch.randn(2, 4, requires_grad = True)+ encoded = BaseModelOutput(last_hidden_state = value)+ expected = model(encoded)+ expected.sum().backward()+ gradient = value.grad.clone()+ model.zero_grad()+ value.grad = None+ actual = torch.compile(model, backend = "eager", fullgraph = True, dynamic = dynamic)(encoded)+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)+ actual.sum().backward()+ torch.testing.assert_close(value.grad, gradient, rtol = 0, atol = 0)+++def test_split_forward_wrapper_deepcopy_uses_the_copy():+ import copy++ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ model = _split_device_metadata(nn.Sequential(nn.Linear(4, 4), nn.Linear(4, 4)))+ assert _remove_same_device_compressed_tensors_offload(model) == 2+ cloned = copy.deepcopy(model)+ x = torch.randn(2, 4, requires_grad = True)+ expected = model(x).detach()+ with torch.no_grad():+ cloned[0].weight.zero_()+ cloned[0].bias.fill_(2)+ cloned[1].weight.fill_(1)+ cloned[1].bias.zero_()+ actual = cloned(x)+ torch.testing.assert_close(actual, torch.full_like(actual, 8))+ torch.testing.assert_close(model(x), expected)+ actual.sum().backward()+ assert cloned[1].weight.grad is not None+ assert all(parameter.grad is None for parameter in model.parameters())+ cloned[0].to("meta")+ out = cloned[0](torch.randn(2, 4, requires_grad = True))+ assert out.device.type == "meta"+ assert out.requires_grad+ assert model[0].weight.device.type == "cpu"++[email protected]("dynamic", [False, True])+def test_split_cleanup_preserves_compilation_and_dtype_conversion(dynamic):+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ model = nn.Sequential(nn.BatchNorm1d(4), nn.Linear(4, 4)).eval()+ reference = nn.Sequential(nn.BatchNorm1d(4), nn.Linear(4, 4)).eval()+ reference.load_state_dict(model.state_dict())+ _split_device_metadata(model)+ assert _remove_same_device_compressed_tensors_offload(model) == 2+ model.double()+ reference.double()+ assert model[0].running_mean.dtype == torch.float64+ x = torch.randn(3, 4, dtype = torch.float64, requires_grad = True)+ expected_x = x.detach().clone().requires_grad_()+ compiled = torch.compile(model, backend = "eager", fullgraph = True, dynamic = dynamic)+ actual = compiled(x)+ expected = reference(expected_x)+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)+ actual.sum().backward()+ expected.sum().backward()+ torch.testing.assert_close(x.grad, expected_x.grad, rtol = 0, atol = 0)+ for param, ref_param in zip(model.parameters(), reference.parameters()):+ torch.testing.assert_close(param.grad, ref_param.grad, rtol = 0, atol = 0)+++def test_split_wrapper_follows_weight_parametrization():+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ model = nn.Sequential(nn.Linear(4, 4, bias = False), nn.Linear(4, 4))+ x = torch.randn(2, 4)+ expected = model(x).detach()+ _split_device_metadata(model)+ assert _remove_same_device_compressed_tensors_offload(model) == 2+ torch.nn.utils.parametrize.register_parametrization(model[0], "weight", nn.Identity())+ torch.testing.assert_close(model(x), expected, rtol = 0, atol = 0)+ model(x).sum().backward()+ assert model[0].parametrizations.weight.original.grad is not None+++def test_split_cleanup_moves_nested_inputs_to_the_live_device():+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ @dataclass+ class Payload:+ tensor: torch.Tensor+ label: str++ class Nested(nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(4), requires_grad = False)++ def forward(self, values, *, payload):+ assert payload.label == "unchanged"+ assert values[1]["other"] is None+ return (values[0] + payload.tensor) * self.weight++ model = _split_device_metadata(nn.ModuleList([nn.Linear(4, 4), Nested()]))+ assert _remove_same_device_compressed_tensors_offload(model) == 2+ model[1].to("meta")+ x = torch.ones(4, requires_grad = True)+ out = model[1]([x, {"other": None}], payload = Payload(x, "unchanged"))+ assert out.device.type == "meta"+ assert out.requires_grad+++def test_split_cleanup_aligns_a_parameterless_residual_container():+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload++ class Residual(nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = nn.Linear(4, 4)++ def forward(self, x):+ return x + self.proj(x)++ model = _split_device_metadata(nn.Sequential(nn.Linear(4, 4), Residual()))+ residual = model[1]+ offload_module_lib.offload_module(residual, torch.device("cpu"), torch.device("cpu"))+ for cache in (residual._parameters, residual._buffers):+ cache.onload_device = cache.offload_device = torch.device("cuda:1")+ assert _remove_same_device_compressed_tensors_offload(model) == 3+ residual.to("meta")+ assert residual(torch.randn(2, 4)).device.type == "meta"+ residual.to_empty(device = "cpu")+ residual.proj.reset_parameters()+ x = torch.randn(2, 4, requires_grad = True)+ residual(x).sum().backward()+ assert x.grad is not None++def test_installed_quantizer_hook_registers_the_unwrap(monkeypatch):quantizer_lib = pytest.importorskip("transformers.quantizers.quantizer_compressed_tensors")cls = quantizer_lib.CompressedTensorsHfQuantizerdiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 068e6f60c5c..56414a025e2 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -3429,25 +3429,107 @@ def enable_composite_gradient_checkpointing(model, verbose = True):return True+def _move_compressed_tensors_inputs(value, device):+ if isinstance(value, torch.Tensor):+ return value.to(device = device)+ if isinstance(value, list):+ return [_move_compressed_tensors_inputs(item, device) for item in value]+ if isinstance(value, tuple):+ return tuple(_move_compressed_tensors_inputs(item, device) for item in value)+ from dataclasses import fields, is_dataclass++ if is_dataclass(value):+ return type(value)(+ **{+ field.name: _move_compressed_tensors_inputs(getattr(value, field.name), device)+ for field in fields(value)+ }+ )+ if isinstance(value, dict):+ return {key: _move_compressed_tensors_inputs(item, device) for key, item in value.items()}+ return value+++def _align_compressed_tensors_inputs(module, args, kwargs):+ reference = next((p for p in module._parameters.values() if p is not None), None)+ if reference is None:+ reference = next((b for b in module._buffers.values() if b is not None), None)+ if reference is None:+ reference = next(module.parameters(), None)+ if reference is None:+ reference = next(module.buffers(), None)+ if reference is None:+ return args, kwargs+ if len(args) == 1 and not kwargs and isinstance(args[0], torch.Tensor):+ if args[0].device == reference.device:+ return None+ return (args[0].to(device = reference.device),), kwargs+ return _move_compressed_tensors_inputs((args, kwargs), reference.device)+++def _compressed_tensors_input_forward(original_forward, module):+ from functools import wraps++ mapping_name, tensor_name = next(+ (+ (mapping, name)+ for mapping in ("_parameters", "_buffers")+ for name, value in getattr(module, mapping).items()+ if value is not None+ ),+ (None, None),+ )++ @wraps(original_forward)+ def forward(module, *args, **kwargs):+ if (+ mapping_name is not None+ and len(args) == 1+ and not kwargs+ and isinstance(args[0], torch.Tensor)+ ):+ reference = getattr(module, mapping_name).get(tensor_name)+ if reference is not None:+ value = args[0]+ if value.device != reference.device:+ value = value.to(device = reference.device)+ return original_forward(module, value)+ aligned = _align_compressed_tensors_inputs(module, args, kwargs)+ if aligned is not None:+ args, kwargs = aligned+ return original_forward(module, *args, **kwargs)++ return forward++def _remove_same_device_compressed_tensors_offload(model):- """Unwrap compressed-tensors >= 0.19 non-offloading OffloadCaches, which torch.compile cannot trace."""try:from compressed_tensors.offload.cache import OffloadCachefrom compressed_tensors.offload.module import remove_module_offloadexcept Exception:return 0- removed = 0+ same_device = []+ onload_devices = set()for module in model.modules():cache = module._parametersif not isinstance(cache, OffloadCache):continueonload = getattr(cache, "onload_device", None)offload = getattr(cache, "offload_device", None)+ if onload is not None:+ onload_devices.add(torch.device(onload))if onload is None or offload is None or torch.device(onload) != torch.device(offload):continue+ same_device.append(module)+ split = len(onload_devices) > 1+ for module in same_device:remove_module_offload(module)- removed += 1- return removed+ if split:+ # The original mover disables autograd and compares data pointers.+ module.forward = _compressed_tensors_input_forward(+ module.forward.__func__, module+ ).__get__(module)+ return len(same_device)def _decompress_compressed_tensors_model(model):