Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Preserve split-device compressed-tensors input transfers

Daniel Hanbbee0dc04fb3
diff --git a/tests/test_compressed_tensors_lazy_decompress_offload.py b/tests/test_compressed_tensors_lazy_decompress_offload.py
index e91ee7c8a19..fd29a737d7d 100644
--- a/tests/test_compressed_tensors_lazy_decompress_offload.py
+++ b/tests/test_compressed_tensors_lazy_decompress_offload.py
@@ -14,12 +14,8 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-"""A compressed-tensors checkpoint loaded with plain transformers in a process where Unsloth is
-imported (say, reloading what Unsloth saved) is decompressed in compressed-tensors' first-forward
-hook, which wraps each module's parameters in an OffloadCache. Unsloth's compiled forwards cannot
-trace that cache, so Unsloth unwraps the same-device ones right after the hook runs."""
-
import types
+from dataclasses import dataclass
import pytest
import torch
@@ -48,9 +44,6 @@ class _Tiny(nn.Module):
class _LazyCompressor:
- """compressed-tensors' own add_decompress_hook / remove_decompression_hook, with a
- decompress_model that does what 0.19's does to the module tree (from_accelerate dispatch)."""
-
add_decompress_hook = ModelCompressor.add_decompress_hook
remove_decompression_hook = ModelCompressor.remove_decompression_hook
@@ -86,7 +79,7 @@ def test_unwrap_hook_runs_after_the_decompress_hook_and_removes_itself():
reference.load_state_dict(model.state_dict())
x = torch.randn(2, 4)
assert mcl._unwrap_offload_after_lazy_decompress(model) is True
- assert mcl._unwrap_offload_after_lazy_decompress(model) is False # idempotent
+ assert mcl._unwrap_offload_after_lazy_decompress(model) is False
out = model(x)
assert _caches(model) == 0
assert not hasattr(model, "ct_decompress_hook")
@@ -114,6 +107,211 @@ def test_real_offload_is_left_alone():
assert _caches(model) == 2
+def _split_device_metadata(model):
+ index = 0
+ for module in model.modules():
+ if module._parameters or module._buffers:
+ offload_module_lib.offload_module(module, torch.device("cpu"), torch.device("cpu"))
+ for cache in (module._parameters, module._buffers):
+ cache.onload_device = torch.device(f"cuda:{index % 2}")
+ cache.offload_device = cache.onload_device
+ index += 1
+ return model
+
+
+def test_split_device_model_keeps_the_input_transfer():
+ model = _split_device_metadata(_Tiny())
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ assert _caches(model) == 0
+ assert not hasattr(model.proj, "_original_forward_func")
+ assert not model.proj._forward_pre_hooks
+ forward = model.proj.forward.__func__
+ assert _remove_same_device_compressed_tensors_offload(model) == 0
+ assert model.proj.forward.__func__ is forward
+
+
+def test_split_cleanup_preserves_encoder_outputs():
+ from transformers import BartConfig, BartForConditionalGeneration
+ from transformers.modeling_outputs import BaseModelOutput
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ config = BartConfig(
+ vocab_size = 32,
+ d_model = 8,
+ encoder_layers = 1,
+ decoder_layers = 1,
+ encoder_attention_heads = 2,
+ decoder_attention_heads = 2,
+ encoder_ffn_dim = 16,
+ decoder_ffn_dim = 16,
+ max_position_embeddings = 8,
+ dropout = 0.0,
+ attention_dropout = 0.0,
+ activation_dropout = 0.0,
+ )
+ model = BartForConditionalGeneration(config).eval()
+ hidden = torch.randn(2, 3, 8, requires_grad = True)
+ encoded = BaseModelOutput(last_hidden_state = hidden)
+ ids = torch.tensor([[2, 3], [2, 4]])
+ expected = model(encoder_outputs = encoded, decoder_input_ids = ids, use_cache = False).logits
+ _split_device_metadata(model)
+ assert _remove_same_device_compressed_tensors_offload(model) > 1
+ actual = model(encoder_outputs = encoded, decoder_input_ids = ids, use_cache = False).logits
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+ actual.square().sum().backward()
+ assert hidden.grad is not None
+
+
[email protected]("dynamic", [False, True])
+def test_split_model_output_fullgraph(dynamic):
+ from transformers.modeling_outputs import BaseModelOutput
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ class Reader(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.eye(4))
+
+ def forward(self, encoded):
+ return encoded.last_hidden_state @ self.weight
+
+ model = _split_device_metadata(nn.Sequential(Reader(), nn.Linear(4, 4)))
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ value = torch.randn(2, 4, requires_grad = True)
+ encoded = BaseModelOutput(last_hidden_state = value)
+ expected = model(encoded)
+ expected.sum().backward()
+ gradient = value.grad.clone()
+ model.zero_grad()
+ value.grad = None
+ actual = torch.compile(model, backend = "eager", fullgraph = True, dynamic = dynamic)(encoded)
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+ actual.sum().backward()
+ torch.testing.assert_close(value.grad, gradient, rtol = 0, atol = 0)
+
+
+def test_split_forward_wrapper_deepcopy_uses_the_copy():
+ import copy
+
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ model = _split_device_metadata(nn.Sequential(nn.Linear(4, 4), nn.Linear(4, 4)))
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ cloned = copy.deepcopy(model)
+ x = torch.randn(2, 4, requires_grad = True)
+ expected = model(x).detach()
+ with torch.no_grad():
+ cloned[0].weight.zero_()
+ cloned[0].bias.fill_(2)
+ cloned[1].weight.fill_(1)
+ cloned[1].bias.zero_()
+ actual = cloned(x)
+ torch.testing.assert_close(actual, torch.full_like(actual, 8))
+ torch.testing.assert_close(model(x), expected)
+ actual.sum().backward()
+ assert cloned[1].weight.grad is not None
+ assert all(parameter.grad is None for parameter in model.parameters())
+ cloned[0].to("meta")
+ out = cloned[0](torch.randn(2, 4, requires_grad = True))
+ assert out.device.type == "meta"
+ assert out.requires_grad
+ assert model[0].weight.device.type == "cpu"
+
+
[email protected]("dynamic", [False, True])
+def test_split_cleanup_preserves_compilation_and_dtype_conversion(dynamic):
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ model = nn.Sequential(nn.BatchNorm1d(4), nn.Linear(4, 4)).eval()
+ reference = nn.Sequential(nn.BatchNorm1d(4), nn.Linear(4, 4)).eval()
+ reference.load_state_dict(model.state_dict())
+ _split_device_metadata(model)
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ model.double()
+ reference.double()
+ assert model[0].running_mean.dtype == torch.float64
+ x = torch.randn(3, 4, dtype = torch.float64, requires_grad = True)
+ expected_x = x.detach().clone().requires_grad_()
+ compiled = torch.compile(model, backend = "eager", fullgraph = True, dynamic = dynamic)
+ actual = compiled(x)
+ expected = reference(expected_x)
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+ actual.sum().backward()
+ expected.sum().backward()
+ torch.testing.assert_close(x.grad, expected_x.grad, rtol = 0, atol = 0)
+ for param, ref_param in zip(model.parameters(), reference.parameters()):
+ torch.testing.assert_close(param.grad, ref_param.grad, rtol = 0, atol = 0)
+
+
+def test_split_wrapper_follows_weight_parametrization():
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ model = nn.Sequential(nn.Linear(4, 4, bias = False), nn.Linear(4, 4))
+ x = torch.randn(2, 4)
+ expected = model(x).detach()
+ _split_device_metadata(model)
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ torch.nn.utils.parametrize.register_parametrization(model[0], "weight", nn.Identity())
+ torch.testing.assert_close(model(x), expected, rtol = 0, atol = 0)
+ model(x).sum().backward()
+ assert model[0].parametrizations.weight.original.grad is not None
+
+
+def test_split_cleanup_moves_nested_inputs_to_the_live_device():
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ @dataclass
+ class Payload:
+ tensor: torch.Tensor
+ label: str
+
+ class Nested(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(4), requires_grad = False)
+
+ def forward(self, values, *, payload):
+ assert payload.label == "unchanged"
+ assert values[1]["other"] is None
+ return (values[0] + payload.tensor) * self.weight
+
+ model = _split_device_metadata(nn.ModuleList([nn.Linear(4, 4), Nested()]))
+ assert _remove_same_device_compressed_tensors_offload(model) == 2
+ model[1].to("meta")
+ x = torch.ones(4, requires_grad = True)
+ out = model[1]([x, {"other": None}], payload = Payload(x, "unchanged"))
+ assert out.device.type == "meta"
+ assert out.requires_grad
+
+
+def test_split_cleanup_aligns_a_parameterless_residual_container():
+ from unsloth.models.loader_utils import _remove_same_device_compressed_tensors_offload
+
+ class Residual(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = nn.Linear(4, 4)
+
+ def forward(self, x):
+ return x + self.proj(x)
+
+ model = _split_device_metadata(nn.Sequential(nn.Linear(4, 4), Residual()))
+ residual = model[1]
+ offload_module_lib.offload_module(residual, torch.device("cpu"), torch.device("cpu"))
+ for cache in (residual._parameters, residual._buffers):
+ cache.onload_device = cache.offload_device = torch.device("cuda:1")
+ assert _remove_same_device_compressed_tensors_offload(model) == 3
+ residual.to("meta")
+ assert residual(torch.randn(2, 4)).device.type == "meta"
+ residual.to_empty(device = "cpu")
+ residual.proj.reset_parameters()
+ x = torch.randn(2, 4, requires_grad = True)
+ residual(x).sum().backward()
+ assert x.grad is not None
+
+
def test_installed_quantizer_hook_registers_the_unwrap(monkeypatch):
quantizer_lib = pytest.importorskip("transformers.quantizers.quantizer_compressed_tensors")
cls = quantizer_lib.CompressedTensorsHfQuantizer
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 068e6f60c5c..56414a025e2 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -3429,25 +3429,107 @@ def enable_composite_gradient_checkpointing(model, verbose = True):
return True
+def _move_compressed_tensors_inputs(value, device):
+ if isinstance(value, torch.Tensor):
+ return value.to(device = device)
+ if isinstance(value, list):
+ return [_move_compressed_tensors_inputs(item, device) for item in value]
+ if isinstance(value, tuple):
+ return tuple(_move_compressed_tensors_inputs(item, device) for item in value)
+ from dataclasses import fields, is_dataclass
+
+ if is_dataclass(value):
+ return type(value)(
+ **{
+ field.name: _move_compressed_tensors_inputs(getattr(value, field.name), device)
+ for field in fields(value)
+ }
+ )
+ if isinstance(value, dict):
+ return {key: _move_compressed_tensors_inputs(item, device) for key, item in value.items()}
+ return value
+
+
+def _align_compressed_tensors_inputs(module, args, kwargs):
+ reference = next((p for p in module._parameters.values() if p is not None), None)
+ if reference is None:
+ reference = next((b for b in module._buffers.values() if b is not None), None)
+ if reference is None:
+ reference = next(module.parameters(), None)
+ if reference is None:
+ reference = next(module.buffers(), None)
+ if reference is None:
+ return args, kwargs
+ if len(args) == 1 and not kwargs and isinstance(args[0], torch.Tensor):
+ if args[0].device == reference.device:
+ return None
+ return (args[0].to(device = reference.device),), kwargs
+ return _move_compressed_tensors_inputs((args, kwargs), reference.device)
+
+
+def _compressed_tensors_input_forward(original_forward, module):
+ from functools import wraps
+
+ mapping_name, tensor_name = next(
+ (
+ (mapping, name)
+ for mapping in ("_parameters", "_buffers")
+ for name, value in getattr(module, mapping).items()
+ if value is not None
+ ),
+ (None, None),
+ )
+
+ @wraps(original_forward)
+ def forward(module, *args, **kwargs):
+ if (
+ mapping_name is not None
+ and len(args) == 1
+ and not kwargs
+ and isinstance(args[0], torch.Tensor)
+ ):
+ reference = getattr(module, mapping_name).get(tensor_name)
+ if reference is not None:
+ value = args[0]
+ if value.device != reference.device:
+ value = value.to(device = reference.device)
+ return original_forward(module, value)
+ aligned = _align_compressed_tensors_inputs(module, args, kwargs)
+ if aligned is not None:
+ args, kwargs = aligned
+ return original_forward(module, *args, **kwargs)
+
+ return forward
+
+
def _remove_same_device_compressed_tensors_offload(model):
- """Unwrap compressed-tensors >= 0.19 non-offloading OffloadCaches, which torch.compile cannot trace."""
try:
from compressed_tensors.offload.cache import OffloadCache
from compressed_tensors.offload.module import remove_module_offload
except Exception:
return 0
- removed = 0
+ same_device = []
+ onload_devices = set()
for module in model.modules():
cache = module._parameters
if not isinstance(cache, OffloadCache):
continue
onload = getattr(cache, "onload_device", None)
offload = getattr(cache, "offload_device", None)
+ if onload is not None:
+ onload_devices.add(torch.device(onload))
if onload is None or offload is None or torch.device(onload) != torch.device(offload):
continue
+ same_device.append(module)
+ split = len(onload_devices) > 1
+ for module in same_device:
remove_module_offload(module)
- removed += 1
- return removed
+ if split:
+ # The original mover disables autograd and compares data pointers.
+ module.forward = _compressed_tensors_input_forward(
+ module.forward.__func__, module
+ ).__get__(module)
+ return len(same_device)
def _decompress_compressed_tensors_model(model):