Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Rename block_swap_layers to offload_layers (#12705)

Daniel Han312d126b015e
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.py
index b8bdbfd7693..28514bfdcd8 100644
--- a/tests/test_block_swap_install.py
+++ b/tests/test_block_swap_install.py
@@ -190,7 +190,7 @@ def test_trim_refuses_what_install_refuses():
def _refusals(fn, needle):
- """Lines of `if <needle ...>: block_swap_layers = refuse_block_swap_load(...)` in `fn`."""
+ """Lines of `if <needle ...>: offload_layers = refuse_block_swap_load(...)` in `fn`."""
return [
n.lineno
for n in ast.walk(fn)
@@ -210,13 +210,13 @@ def _peft_signature_and_calls(path):
@pytest.mark.parametrize("path", ["llama.py", "vision.py"])
-def test_block_swap_layers_is_last_so_positional_callers_keep_their_slots(path):
+def test_offload_layers_is_last_so_positional_callers_keep_their_slots(path):
fn = _peft_signature_and_calls(path)
- assert [a.arg for a in fn.args.args[-2:]] == ["block_swap_layers", "checkpoint_skip_layers"]
+ assert [a.arg for a in fn.args.args[-2:]] == ["offload_layers", "checkpoint_skip_layers"]
assert fn.args.kwarg is not None
-def test_new_model_route_forwards_block_swap_layers():
+def test_new_model_route_forwards_offload_layers():
fn = _peft_signature_and_calls("llama.py")
forwarded = [
call
@@ -228,7 +228,7 @@ def test_new_model_route_forwards_block_swap_layers():
]
assert forwarded, "llama get_peft_model no longer delegates to FastBaseModel"
for call in forwarded:
- assert {"block_swap_layers", "checkpoint_skip_layers"} <= {k.arg for k in call.keywords}
+ assert {"offload_layers", "checkpoint_skip_layers"} <= {k.arg for k in call.keywords}
@pytest.mark.parametrize("path", ["llama.py", "gemma.py", "gemma2.py"])
@@ -676,7 +676,7 @@ def test_load_time_swappers_allocate_the_planned_prefetch_depth():
assert built == [1] and ns["planned_prefetch_depth"](None) == 2
# Both load paths hand the planner's depth to the swapper they build.
for path, call in (
- ("llama.py", "attach_block_swap_layers"),
+ ("llama.py", "attach_offload_layers"),
("vision.py", "finish_block_swap_load"),
):
src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
@@ -688,3 +688,23 @@ def test_load_time_swappers_allocate_the_planned_prefetch_depth():
assert "planned_prefetch_depth(device_map_planner_kwargs)" in ast.get_source_segment(
src, node
)
+
+
+def test_block_swap_layers_still_works_as_the_old_name_of_offload_layers():
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ fn = next(
+ n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name == "legacy_offload_layers"
+ )
+ ns = {}
+ exec(compile(ast.Module(body = [fn], type_ignores = []), UTILS, "exec"), ns)
+ legacy = ns["legacy_offload_layers"]
+ kwargs = {"block_swap_layers": "auto"}
+ assert legacy(kwargs, None) == "auto" and kwargs == {}
+ # The new name wins whenever it was given, an explicit 0 included.
+ assert legacy({"block_swap_layers": 4}, 2) == 2
+ assert legacy({"block_swap_layers": 4}, 0) == 0
+ assert legacy({}, None) == 0 and legacy({}, 3) == 3
+ # Every entry point maps the old name before reading the new one.
+ for path, count in (("llama.py", 2), ("vision.py", 2)):
+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
+ assert src.count("legacy_offload_layers(kwargs, ") == count, path
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 467ee63773b..023abd5e081 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -92,12 +92,13 @@ __all__ = [
"usable_cuda_bytes",
"skip_checkpointing",
"refuse_block_swap_load",
+ "legacy_offload_layers",
"block_swap_load_device",
"begin_block_swap_load",
"finish_block_swap_load",
"planned_prefetch_depth",
"trim_config_for_block_swap",
- "attach_block_swap_layers",
+ "attach_offload_layers",
"skip_swapped_checkpoint_keys",
"get_moe_target_parameters",
"get_moe_target_modules",
@@ -187,7 +188,7 @@ except ImportError: # unsloth_zoo predates loading any architecture straight to
load_layers_to_host = None
try:
from unsloth_zoo.block_swap import auto_swap_indices, estimate_training_reserve_bytes
-except ImportError: # unsloth_zoo predates block_swap_layers = "auto"
+except ImportError: # unsloth_zoo predates offload_layers = "auto"
auto_swap_indices = estimate_training_reserve_bytes = None
from unsloth_zoo.gradient_checkpointing import (
Unsloth_Offloaded_Gradient_Checkpointer,
@@ -6123,20 +6124,20 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:
def _check_block_swap(model_or_config):
if BlockSwap is None:
raise ImportError(
- "Unsloth: block_swap_layers needs a newer unsloth_zoo. "
+ "Unsloth: offload_layers needs a newer unsloth_zoo. "
"Run `pip install --upgrade unsloth_zoo`."
)
if is_moe_model(model_or_config):
print(
- "Unsloth: block_swap_layers on an MoE model copies every expert across PCIe "
+ "Unsloth: offload_layers on an MoE model copies every expert across PCIe "
"while only the routed ones compute, so steps can be copy-bound."
)
if not torch.cuda.is_available():
# Prefetch runs on CUDA/HIP streams; XPU, NPU and CPU have none.
- raise ValueError("Unsloth: block_swap_layers needs a CUDA or ROCm GPU.")
+ raise ValueError("Unsloth: offload_layers needs a CUDA or ROCm GPU.")
if is_integrated_unified_memory_gpu():
raise ValueError(
- "Unsloth: block_swap_layers has nothing to swap to on a unified-memory "
+ "Unsloth: offload_layers has nothing to swap to on a unified-memory "
"GPU; host and device already share the same RAM."
)
@@ -6151,14 +6152,22 @@ def _new_block_swap(layers, n, *args, placement, **kwargs):
return BlockSwap(layers, n, *args, **kwargs)
-def refuse_block_swap_load(block_swap_layers, reason):
+def legacy_offload_layers(kwargs, offload_layers = None):
+ """`offload_layers`, or its original name `block_swap_layers` from `kwargs` when it was not given (0 = off)."""
+ legacy = kwargs.pop("block_swap_layers", None)
+ if offload_layers is None:
+ return 0 if legacy is None else legacy
+ return offload_layers
+
+
+def refuse_block_swap_load(offload_layers, reason):
"""A load-to-host restriction: `"auto"` falls back to loading onto the GPU (0), a count raises."""
- if block_swap_layers == "auto":
+ if offload_layers == "auto":
print(
- f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: loading into host RAM {reason}"
+ f"Unsloth: offload_layers = 'auto' loads every layer onto the GPU: loading into host RAM {reason}"
)
return 0
- raise ValueError(f"Unsloth: from_pretrained(block_swap_layers = ...) {reason}")
+ raise ValueError(f"Unsloth: from_pretrained(offload_layers = ...) {reason}")
def block_swap_load_device(device_map):
@@ -6182,18 +6191,18 @@ def block_swap_load_device(device_map):
def begin_block_swap_load(
- block_swap_layers,
+ offload_layers,
device_map,
embeddings = False,
):
- """from_pretrained(block_swap_layers = N) on any architecture: the context the weight load runs in,
+ """from_pretrained(offload_layers = N) on any architecture: the context the weight load runs in,
moving N decoder layers (and with `embeddings`, the large extra token tables) to host RAM as each
finishes loading. nullcontext when N is 0."""
- if not block_swap_layers and not embeddings:
+ if not offload_layers and not embeddings:
return contextlib.nullcontext()
if embeddings:
- return load_layers_to_host(block_swap_layers, placement = "spread", embeddings = True)
- return load_layers_to_host(block_swap_layers, placement = "spread")
+ return load_layers_to_host(offload_layers, placement = "spread", embeddings = True)
+ return load_layers_to_host(offload_layers, placement = "spread")
def planned_prefetch_depth(device_map_planner_kwargs):
@@ -6358,7 +6367,7 @@ def _auto_block_swap_indices(model, prefetch_depth):
"""Layers to swap so each GPU keeps a training step's reserve free; [] when it already does."""
if auto_swap_indices is None:
raise ImportError(
- "Unsloth: block_swap_layers = 'auto' needs a newer unsloth_zoo. "
+ "Unsloth: offload_layers = 'auto' needs a newer unsloth_zoo. "
"Run `pip install --upgrade unsloth_zoo`."
)
layers = find_decoder_layers(model)
@@ -6369,38 +6378,38 @@ def _auto_block_swap_indices(model, prefetch_depth):
indices, left = auto_swap_indices(layers, reserve, prefetch_depth)
if not indices:
print(
- f"Unsloth: block_swap_layers = 'auto' swaps nothing: every GPU keeps the "
+ f"Unsloth: offload_layers = 'auto' swaps nothing: every GPU keeps the "
f"{reserve / 2**30:.2f} GiB a {seq_len}-token step needs."
)
else:
print(
- f"Unsloth: block_swap_layers = 'auto' keeps {len(indices)} of {len(layers)} decoder layers "
+ f"Unsloth: offload_layers = 'auto' keeps {len(indices)} of {len(layers)} decoder layers "
f"in host RAM so {reserve / 2**30:.2f} GiB stays free for a {seq_len}-token step."
)
if left > 0:
print(
- f"Unsloth: block_swap_layers = 'auto' is still {left / 2**30:.2f} GiB short; "
- "lower max_seq_length, or load with from_pretrained(block_swap_layers = 'auto')."
+ f"Unsloth: offload_layers = 'auto' is still {left / 2**30:.2f} GiB short; "
+ "lower max_seq_length, or load with from_pretrained(offload_layers = 'auto')."
)
return indices
def install_block_swap(
model,
- block_swap_layers = 0,
+ offload_layers = 0,
prefetch_depth = 2,
use_gradient_checkpointing = "unsloth",
):
- """Stream frozen decoder blocks from pinned host RAM: `block_swap_layers` of them, or "auto" for
+ """Stream frozen decoder blocks from pinned host RAM: `offload_layers` of them, or "auto" for
as few as keep a training step's reserve free on every GPU; off at 0."""
existing = getattr(model, "_unsloth_block_swap", None)
if existing is None and (
- not block_swap_layers or (block_swap_layers != "auto" and block_swap_layers <= 0)
+ not offload_layers or (offload_layers != "auto" and offload_layers <= 0)
):
return None
if not use_gradient_checkpointing:
raise ValueError(
- "Unsloth: block_swap_layers needs use_gradient_checkpointing. Without "
+ "Unsloth: offload_layers needs use_gradient_checkpointing. Without "
"it every swapped block stays on the card until backward, so there is "
"nothing to save and the slot pool runs dry mid-forward."
)
@@ -6408,22 +6417,22 @@ def install_block_swap(
return existing
if getattr(model, "vllm_engine", None) is not None:
raise ValueError(
- "Unsloth: block_swap_layers cannot be combined with fast_inference = True, "
+ "Unsloth: offload_layers cannot be combined with fast_inference = True, "
"since evicted weights would sync to vLLM as empty tensors."
)
- if block_swap_layers == "auto":
+ if offload_layers == "auto":
# Nothing to swap to without a discrete CUDA / ROCm card: auto means no swap there.
if not torch.cuda.is_available() or is_integrated_unified_memory_gpu():
return None
if BlockSwap is None:
_check_block_swap(model)
- block_swap_layers = _auto_block_swap_indices(model, prefetch_depth)
- if not block_swap_layers:
+ offload_layers = _auto_block_swap_indices(model, prefetch_depth)
+ if not offload_layers:
return None
_check_block_swap(model)
layers = find_decoder_layers(model)
# Spaced evenly, each copy hides behind several layers of compute instead of one.
- swapper = _new_block_swap(layers, block_swap_layers, prefetch_depth, placement = "spread")
+ swapper = _new_block_swap(layers, offload_layers, prefetch_depth, placement = "spread")
# On the layer list too: the fast decode loop only sees the inner model.
layers._unsloth_block_swap = swapper
model._unsloth_block_swap = swapper
@@ -6476,9 +6485,7 @@ def _checkpoint_tensors(
for shard in shards:
path = _get(shard)
if path is None:
- raise RuntimeError(
- f"Unsloth: block_swap_layers could not find {shard} for {model_name}."
- )
+ raise RuntimeError(f"Unsloth: offload_layers could not find {shard} for {model_name}.")
h = safe_open(path, framework = "pt", device = "cpu")
handles.append(h)
for key in h.keys():
@@ -6486,18 +6493,18 @@ def _checkpoint_tensors(
return tensors, handles
-def trim_config_for_block_swap(config, block_swap_layers):
+def trim_config_for_block_swap(config, offload_layers):
"""Trim the swapped tail off the config; returns the originals to restore, or None."""
- if not block_swap_layers or block_swap_layers <= 0:
+ if not offload_layers or offload_layers <= 0:
return None
if build_host_layers is None:
raise ImportError(
- "Unsloth: from_pretrained(block_swap_layers = ...) needs a newer unsloth_zoo. "
+ "Unsloth: from_pretrained(offload_layers = ...) needs a newer unsloth_zoo. "
"Run `pip install --upgrade unsloth_zoo`."
)
_check_block_swap(config)
total = config.num_hidden_layers
- n = min(int(block_swap_layers), total - 1)
+ n = min(int(offload_layers), total - 1)
saved = {"num_hidden_layers": total}
for key, value in list(vars(config).items()):
if isinstance(value, (list, tuple)) and len(value) == total:
@@ -6538,7 +6545,7 @@ def skip_swapped_checkpoint_keys(saved, kept):
return undo
-def attach_block_swap_layers(
+def attach_offload_layers(
model,
saved,
model_name,
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 8cd6fa87fef..43c74d1f63c 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2734,7 +2734,7 @@ class FastLlamaModel:
# Respect a user-provided config so it is the single config object used everywhere below; else
# HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.
user_config = kwargs.pop("config", None)
- block_swap_layers = kwargs.pop("block_swap_layers", 0)
+ offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))
offload_embedding = kwargs.pop("offload_embedding", False)
if offload_embedding and fast_inference:
if offload_embedding != OFFLOAD_EMBEDDING_AUTO:
@@ -2745,21 +2745,19 @@ class FastLlamaModel:
_offload_embedding_mode = offload_embedding
# None: no memory plan; True / False: the block swap planner decided.
_embedding_needed = None
- if block_swap_layers and kwargs.get("state_dict") is not None:
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers,
+ if offload_layers and kwargs.get("state_dict") is not None:
+ offload_layers = refuse_block_swap_load(
+ offload_layers,
"does not take a state_dict; save it as a safetensors checkpoint and load that.",
)
- if block_swap_layers and kwargs.get("quantization_config") is not None:
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers,
+ if offload_layers and kwargs.get("quantization_config") is not None:
+ offload_layers = refuse_block_swap_load(
+ offload_layers,
"does not take a quantization_config; pass load_in_4bit = True instead.",
)
- if block_swap_layers and (
- kwargs.get("gguf_file") or kwargs.get("use_safetensors") is False
- ):
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers,
+ if offload_layers and (kwargs.get("gguf_file") or kwargs.get("use_safetensors") is False):
+ offload_layers = refuse_block_swap_load(
+ offload_layers,
"needs a safetensors checkpoint; it does not support gguf_file or use_safetensors = False.",
)
if user_config is not None:
@@ -2943,14 +2941,14 @@ class FastLlamaModel:
# checkpoint left unarmed (vLLM reads it itself) must not reach vLLM as a bitsandbytes load.
if not (_explicit_bnb_4bit and _checked_4bit):
load_in_4bit, load_in_8bit = _checked_4bit, _checked_8bit
- if block_swap_layers and load_in_8bit:
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."
+ if offload_layers and load_in_8bit:
+ offload_layers = refuse_block_swap_load(
+ offload_layers, "supports 16-bit and 4-bit loads, not load_in_8bit."
)
- if block_swap_layers and _ckpt_quant_method not in (None, "bitsandbytes"):
+ if offload_layers and _ckpt_quant_method not in (None, "bitsandbytes"):
# The host tail is rebuilt as dense or bnb 4-bit layers; packed formats would not survive it.
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers,
+ offload_layers = refuse_block_swap_load(
+ offload_layers,
f"does not support {_ckpt_quant_method} checkpoints; use a bitsandbytes or 16-bit checkpoint.",
)
from .modelopt_fp8 import (
@@ -3020,12 +3018,12 @@ class FastLlamaModel:
"load adds, so the plan would size dense weights at 4bit"
)
- if block_swap_layers and (fast_inference or num_labels is not None):
- block_swap_layers = refuse_block_swap_load(
- block_swap_layers, "does not support fast_inference or classification heads."
+ if offload_layers and (fast_inference or num_labels is not None):
+ offload_layers = refuse_block_swap_load(
+ offload_layers, "does not support fast_inference or classification heads."
)
- if block_swap_layers == "auto":
- block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(
+ if offload_layers == "auto":
+ offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(
requested_device_map(device_map),
model_name,
max_seq_length = max_seq_length,
@@ -3141,12 +3139,12 @@ class FastLlamaModel:
kwargs = add_dtype_kwargs(dtype, kwargs)
- if block_swap_layers:
+ if offload_layers:
import copy as _copy
# Trim a copy: HF deep-copies config= anyway, so a trimmed caller config would stay short.
model_config = _copy.deepcopy(model_config)
- _block_swap_saved = trim_config_for_block_swap(model_config, block_swap_layers)
+ _block_swap_saved = trim_config_for_block_swap(model_config, offload_layers)
_undo_block_swap_keys = skip_swapped_checkpoint_keys(
_block_swap_saved, model_config.num_hidden_layers
)
@@ -3381,7 +3379,7 @@ class FastLlamaModel:
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
- attach_block_swap_layers(
+ attach_offload_layers(
model,
_block_swap_saved,
model_name,
@@ -3651,10 +3649,11 @@ class FastLlamaModel:
qat_scheme = None,
target_parameters = None, # For MoE expert layers (nn.Parameter)
ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)
- block_swap_layers = 0,
+ offload_layers = None,
checkpoint_skip_layers = 0,
**kwargs,
):
+ offload_layers = legacy_offload_layers(kwargs, offload_layers)
# The flag reflects the LAST load, not this model.
_text_seq2seq = _is_text_seq2seq_config(getattr(model, "config", None))
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1" or _text_seq2seq:
@@ -3690,7 +3689,7 @@ class FastLlamaModel:
temporary_location = temporary_location,
target_parameters = target_parameters,
ensure_weight_tying = ensure_weight_tying,
- block_swap_layers = block_swap_layers,
+ offload_layers = offload_layers,
checkpoint_skip_layers = checkpoint_skip_layers,
**kwargs,
)
@@ -3819,7 +3818,7 @@ class FastLlamaModel:
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
model = _exclude_rope_inv_freq_from_ddp(model)
install_block_swap(
- model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
)
skip_checkpointing(model, checkpoint_skip_layers)
return model
@@ -4127,7 +4126,7 @@ class FastLlamaModel:
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
offload_embedding_if_tight(model)
install_block_swap(
- model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
)
skip_checkpointing(model, checkpoint_skip_layers)
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 31d88838afc..ad9119cc2b8 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -668,7 +668,7 @@ def resolve_auto_block_swap(
placement = "tail",
**config_kwargs,
):
- """`from_pretrained(block_swap_layers = "auto")`: `(layers, device_map, embedding)`, the trailing
+ """`from_pretrained(offload_layers = "auto")`: `(layers, device_map, embedding)`, the trailing
decoder layers to build in host RAM so the rest plus a training step's reserve fits, the map to load
the rest with, and whether to move the input embedding to host RAM first (one GPU, when
`offload_embedding` allows; None when nothing was planned). 0, the map unchanged and no move when
@@ -676,7 +676,7 @@ def resolve_auto_block_swap(
card through the multi-GPU planner; anything else sizes the one card the load uses."""
def _none(reason):
- print(f"Unsloth: block_swap_layers = 'auto' loads every layer onto the GPU: {reason}.")
+ print(f"Unsloth: offload_layers = 'auto' loads every layer onto the GPU: {reason}.")
return 0, device_map, None
if skip_reason is not None:
@@ -752,7 +752,7 @@ def resolve_auto_block_swap(
)
print(
"Unsloth: "
- + plan.describe().splitlines()[0].replace("block swap:", "block_swap_layers = 'auto':")
+ + plan.describe().splitlines()[0].replace("block swap:", "offload_layers = 'auto':")
)
embedding = bool(getattr(plan, "offload_embedding", False))
if not plan.layers:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 82f683e7a52..bec44dc5961 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2627,24 +2627,24 @@ class FastBaseModel:
user_config = kwargs.pop("config", None)
if auto_config is None and user_config is not None:
auto_config = user_config
- _block_swap_layers = kwargs.pop("block_swap_layers", 0)
- if _block_swap_layers and load_layers_to_host is None:
- _block_swap_layers = refuse_block_swap_load(
- _block_swap_layers,
+ _offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))
+ if _offload_layers and load_layers_to_host is None:
+ _offload_layers = refuse_block_swap_load(
+ _offload_layers,
"needs a newer unsloth_zoo (`pip install --upgrade unsloth_zoo`); "
- "get_peft_model(block_swap_layers = ...) still swaps once the model is on the card.",
+ "get_peft_model(offload_layers = ...) still swaps once the model is on the card.",
)
- if _block_swap_layers and (fast_inference or full_finetuning):
- _block_swap_layers = refuse_block_swap_load(
- _block_swap_layers,
+ if _offload_layers and (fast_inference or full_finetuning):
+ _offload_layers = refuse_block_swap_load(
+ _offload_layers,
"streams frozen LoRA base weights, so it cannot be combined with "
"fast_inference or full_finetuning.",
)
- if _block_swap_layers and (
+ if _offload_layers and (
not torch.cuda.is_available() or is_integrated_unified_memory_gpu()
):
- _block_swap_layers = refuse_block_swap_load(
- _block_swap_layers,
+ _offload_layers = refuse_block_swap_load(
+ _offload_layers,
"needs a discrete CUDA or ROCm GPU; unified memory has no separate RAM to load into.",
)
@@ -3092,14 +3092,14 @@ class FastBaseModel:
raise RuntimeError(
"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!"
)
- if _block_swap_layers and block_swap_load_device(device_map) is None:
- _block_swap_layers = refuse_block_swap_load(
- _block_swap_layers,
- "needs every layer on one GPU; pass block_swap_layers to get_peft_model instead.",
+ if _offload_layers and block_swap_load_device(device_map) is None:
+ _offload_layers = refuse_block_swap_load(
+ _offload_layers,
+ "needs every layer on one GPU; pass offload_layers to get_peft_model instead.",
)
_embedding_needed = None
- if _block_swap_layers == "auto":
- _block_swap_layers, device_map, _embedding_needed = resolve_auto_block_swap(
+ if _offload_layers == "auto":
+ _offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(
requested_device_map(device_map),
model_name,
max_seq_length = max_seq_length,
@@ -3136,7 +3136,7 @@ class FastBaseModel:
),
)
_block_swap_device = block_swap_load_device(device_map)
- if (_block_swap_layers or _embedding_needed) and _block_swap_device is not None:
+ if (_offload_layers or _embedding_needed) and _block_swap_device is not None:
# A placement strategy would spill the host-bound weights to the CPU, which 4-bit loads refuse.
device_map = {"": _block_swap_device}
_block_swap_state = None
@@ -3328,7 +3328,7 @@ class FastBaseModel:
raise_handler = RaiseUninitialized()
try:
- # get_peft_model(block_swap_layers = "auto") may move it later, unless the caller said no.
+ # get_peft_model(offload_layers = "auto") may move it later, unless the caller said no.
_offload_embedding_mode = False if fast_inference else offload_embedding
if offload_embedding and fast_inference:
if offload_embedding != OFFLOAD_EMBEDDING_AUTO:
@@ -3360,7 +3360,7 @@ class FastBaseModel:
):
try:
with begin_block_swap_load(
- _block_swap_layers, device_map, embeddings = bool(_embedding_needed)
+ _offload_layers, device_map, embeddings = bool(_embedding_needed)
) as (_block_swap_state):
model = auto_model.from_pretrained(
model_name,
@@ -3959,10 +3959,11 @@ class FastBaseModel:
target_parameters = None, # For MoE expert layers (nn.Parameter)
ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)
finetune_audio_layers = False, # placed last to preserve existing positional argument order
- block_swap_layers = 0,
+ offload_layers = None,
checkpoint_skip_layers = 0,
**kwargs,
):
+ offload_layers = legacy_offload_layers(kwargs, offload_layers)
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
# Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the detector here too (idempotent).
@@ -4333,7 +4334,7 @@ class FastBaseModel:
module.max_seq_length = max_seq_length
offload_embedding_if_tight(model)
install_block_swap(
- model, block_swap_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
)
skip_checkpointing(model, checkpoint_skip_layers)
for _ in range(3):