Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio / Desktop: Block Swap settings, a VRAM budget, and a live layer panel on the training page (#12996)

Leo Borcherdingd953dcaa07ce
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 6bec5077ca6..7b3efd16a46 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -363,6 +363,9 @@ class UnslothTrainer:
self.is_audio_vlm = False
self._audio_type = None
self._use_gradient_checkpointing = "unsloth"
+ self._offload_layers = 0
+ self._prefetch_depth = 2
+ self._offload_plan_shape = {}
# True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.
self._audio_type_known = True
self._is_dataset_audio = False
@@ -571,6 +574,52 @@ class UnslothTrainer:
except Exception as e:
logger.error(f"Error in progress callback: {e}")
+ def _offload_load_kwargs(self, device_map = None) -> dict:
+ # Loading the offloaded layers straight into host RAM is what lets a model larger than the card load at all.
+ if not self._offload_layers:
+ return {}
+ # A fixed count loads to host only onto one card (core raises on a multi-GPU map); get_peft_model swaps it there.
+ if self._offload_layers != "auto" and device_map in ("unsloth_balanced", "balanced"):
+ return {}
+ return {
+ "offload_layers": self._offload_layers,
+ # Auto plans at load; without the run's batch and rank it sizes for batch 1, rank 16.
+ "device_map_planner_kwargs": {
+ "prefetch_depth": self._prefetch_depth,
+ **self._offload_plan_shape,
+ },
+ }
+
+ def _offload_peft_kwargs(self) -> dict:
+ if not self._offload_layers:
+ return {}
+ return {"offload_layers": self._offload_layers, "prefetch_depth": self._prefetch_depth}
+
+ def _offload_snapshot(self) -> Optional[dict]:
+ """Where each decoder layer is and what the last steps' copies cost, for the live panel."""
+ swapper = (
+ getattr(self.model, "_unsloth_block_swap", None) if self.model is not None else None
+ )
+ stats = getattr(swapper, "stats", None)
+ if stats is None:
+ return None
+ try:
+ snap = stats()
+ # JSON object keys must be strings.
+ snap["state"] = {str(k): v for k, v in snap["state"].items()}
+ if torch.cuda.is_available():
+ device = getattr(swapper, "device", None)
+ snap["vram_allocated_bytes"] = torch.cuda.memory_allocated(device)
+ snap["vram_peak_bytes"] = torch.cuda.max_memory_allocated(device)
+ snap["vram_total_bytes"] = torch.cuda.get_device_properties(device).total_memory
+ get_fraction = getattr(torch.cuda, "get_per_process_memory_fraction", None)
+ if get_fraction is not None:
+ snap["vram_fraction"] = get_fraction(device)
+ return snap
+ except Exception as exc:
+ logger.debug("offload stats unavailable: %s", exc)
+ return None
+
def _create_progress_callback(self):
"""Create a TrainerCallback for progress tracking. Reused by all training branches."""
from transformers import TrainerCallback
@@ -689,6 +738,7 @@ class UnslothTrainer:
num_tokens = num_tokens,
eval_loss = logs.get("eval_loss", None),
is_run_summary = is_run_summary,
+ offload = trainer_ref._offload_snapshot(),
status_message = "",
)
@@ -902,8 +952,15 @@ class UnslothTrainer:
model_revision: Optional[str] = None,
use_gradient_checkpointing: Union[str, bool] = "unsloth",
on_model_resolved: Optional[Callable[[str], None]] = None,
+ offload_layers: Union[int, str] = 0,
+ prefetch_depth: Union[int, str] = 2,
+ offload_plan_shape: Optional[dict] = None,
) -> bool:
"""Load model for training (supports both text and vision models)"""
+ # Offloading streams frozen base weights, so it has nothing to do in a full finetune.
+ self._offload_layers = 0 if full_finetuning else (offload_layers or 0)
+ self._prefetch_depth = prefetch_depth or 2
+ self._offload_plan_shape = {k: v for k, v in (offload_plan_shape or {}).items() if v}
self.load_in_4bit = load_in_4bit
self.trust_remote_code = trust_remote_code
# The loader installs the checkpointing implementation; a full finetune never reinstalls it.
@@ -1123,6 +1180,7 @@ class UnslothTrainer:
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
+ **self._offload_load_kwargs(device_map),
)
logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")
@@ -1199,6 +1257,7 @@ class UnslothTrainer:
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
+ **self._offload_load_kwargs(device_map),
)
logger.info("Loaded audio VLM model (FastModel)")
@@ -1216,6 +1275,7 @@ class UnslothTrainer:
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
+ **self._offload_load_kwargs(device_map),
)
logger.info("Loaded vision model")
@@ -1245,6 +1305,7 @@ class UnslothTrainer:
use_exact_model_name = model_revision is not None,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
+ **self._offload_load_kwargs(device_map),
)
logger.info("Loaded text model")
@@ -1297,6 +1358,9 @@ class UnslothTrainer:
model_revision = model_revision,
use_gradient_checkpointing = use_gradient_checkpointing,
on_model_resolved = on_model_resolved,
+ offload_layers = offload_layers,
+ prefetch_depth = prefetch_depth,
+ offload_plan_shape = offload_plan_shape,
)
error_msg = str(e)
error_lower = error_msg.lower()
@@ -1447,6 +1511,7 @@ class UnslothTrainer:
use_rslora = use_rslora,
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
+ **self._offload_peft_kwargs(),
)
if self.is_audio_vlm:
peft_kwargs.update(
@@ -1524,6 +1589,7 @@ class UnslothTrainer:
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
modules_to_save = modules_to_save,
+ **self._offload_peft_kwargs(),
)
else:
logger.info(f"Text model LoRA configuration:")
@@ -1544,6 +1610,7 @@ class UnslothTrainer:
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
modules_to_save = modules_to_save,
+ **self._offload_peft_kwargs(),
)
if self.should_stop:
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 4dce66fad2f..df28984429f 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -303,6 +303,9 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]:
"lora_dropout": values.get("lora_dropout", 0.0),
"target_modules": values.get("target_modules"),
"gradient_checkpointing": values.get("gradient_checkpointing", "unsloth"),
+ "offload_layers": values.get("offload_layers") or 0,
+ "offload_vram_gb": values.get("offload_vram_gb"),
+ "prefetch_depth": values.get("prefetch_depth") or 2,
"use_rslora": values.get("use_rslora", False),
"use_loftq": values.get("use_loftq", False),
"use_dora": values.get("use_dora", False),
@@ -696,6 +699,8 @@ class TrainingProgress:
num_tokens: Optional[int] = None
eval_loss: Optional[float] = None
peak_memory_gb: Optional[float] = None
+ # BlockSwap.stats() from the last logged step, for the live offload panel.
+ offload: Optional[dict] = None
output_dir: Optional[str] = None
# The end-of-run record has no step loss, so the progress filter would drop it, and with it the only
# elapsed time that includes the final evaluation, checkpoint save and best-model reload.
@@ -3192,6 +3197,9 @@ class TrainingBackend:
self._progress.peak_memory_gb = float(_peak)
except (TypeError, ValueError):
pass
+ # A step without stats (eval, status) keeps the last snapshot, so the panel does not blank.
+ if event.get("offload"):
+ self._progress.offload = event["offload"]
self._progress.is_training = True
status = event.get("status_message", "")
if status:
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 8131d037d45..8d79e41debd 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -1636,6 +1636,48 @@ def _parse_mem_fraction_env(env_value: str | None) -> float | None:
return override if 0.0 < override <= 1.0 else None
+def _training_vram_budget_fraction(
+ budget_gb: float | None,
+ denominator_bytes: int,
+ current: float = 1.0,
+) -> float | None:
+ """The memory fraction that holds this process to ``budget_gb``, never looser than ``current``
+ (the OOM guard's cap); None when there is no budget or no total to divide by."""
+ try:
+ budget = float(budget_gb)
+ except (TypeError, ValueError):
+ return None
+ if not (budget > 0) or denominator_bytes <= 0:
+ return None
+ return min(current, budget * 1024**3 / denominator_bytes)
+
+
+def _offload_plan_shape(config: dict) -> dict:
+ """The per-device batch and LoRA rank an Auto offload plan sizes its training reserve for."""
+ return {"batch_size": config.get("batch_size"), "lora_rank": config.get("lora_r")}
+
+
+def _visible_gpu_count() -> int:
+ """GPUs the load spreads over when no ids were resolved (a UUID / MIG mask is inherited as is)."""
+ try:
+ import torch
+ return torch.cuda.device_count() if torch.cuda.is_available() else 0
+ except Exception:
+ return 0
+
+
+def _with_vram_budget_hint(config: dict, message: str) -> str:
+ """Point a run that does not fit at its own VRAM budget, the one cause the generic advice omits."""
+ budget = config.get("offload_vram_gb")
+ lower = (message or "").lower()
+ if not budget or not any(k in lower for k in ("out of memory", "out of vram", "does not fit")):
+ return message
+ return (
+ f"{message}\nThis run is capped at a {budget:g} GiB VRAM budget. Raise or clear the "
+ "VRAM budget under Training Hyperparameters > Memory."
+ )
+
+
def _allocator_divides_by_props_total(torch_version: str | None) -> bool:
"""Whether ``set_per_process_memory_fraction`` scales ``props.total_memory``. c10's
``CUDACachingAllocator::setMemoryFraction`` caps at ``fraction * device_prop.totalGlobalMem``
@@ -4191,6 +4233,64 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
)
return
+ # Offload layers sizes "auto" to what the allocator may use, so a budget makes the run fit in it,
+ # and two runs on one card can each take their share.
+ # ── 2b. Training VRAM budget ──
+ _budget_gb = config.get("offload_vram_gb")
+ # Only "auto" sizes to a budget, and only LoRA runs outside decision / embedding offload.
+ _wants_budget = bool(
+ _budget_gb
+ and config.get("offload_layers") == "auto"
+ and config.get("training_type", "LoRA/QLoRA") in ("LoRA/QLoRA", "Continued Pretraining")
+ and not config.get("is_decision")
+ and not config.get("is_embedding")
+ )
+ # Vision / audio loads on several GPUs cannot offload at load, and LoRA setup allocates adapters before it swaps:
+ # a cap would OOM either step, so those runs offload Auto uncapped.
+ _budget_unsupported = (
+ _wants_budget
+ and (len(gpu_ids) if gpu_ids else _visible_gpu_count()) > 1
+ and bool(config.get("is_dataset_image") or config.get("is_dataset_audio"))
+ )
+
+ if _budget_unsupported:
+ logger.info(
+ "Training VRAM budget not applied: multi-GPU vision / audio runs offload at LoRA setup"
+ )
+ # An out-of-memory error then must not blame a cap that was never set.
+ config["offload_vram_gb"] = None
+ elif _wants_budget:
+ try:
+ import torch as _torch_budget
+ if _torch_budget.cuda.is_available():
+ for _b_index in range(_torch_budget.cuda.device_count()):
+ _b_props = _torch_budget.cuda.get_device_properties(_b_index)
+ _b_denominator = int(getattr(_b_props, "total_memory", 0) or 0)
+ if not _allocator_divides_by_props_total(
+ getattr(_torch_budget, "__version__", "")
+ ):
+ _b_denominator = int(_torch_budget.cuda.mem_get_info(_b_index)[1])
+ _get_fraction = getattr(
+ _torch_budget.cuda, "get_per_process_memory_fraction", None
+ )
+ _b_current = _get_fraction(_b_index) if _get_fraction is not None else 1.0
+ _b_fraction = _training_vram_budget_fraction(
+ _budget_gb, _b_denominator, _b_current
+ )
+ if _b_fraction is None:
+ continue
+ _torch_budget.cuda.set_per_process_memory_fraction(_b_fraction, _b_index)
+ logger.info(
+ "Training VRAM budget: set_per_process_memory_fraction(%.4f, cuda:%d), "
+ "%.1f GiB of %.1f GiB",
+ _b_fraction,
+ _b_index,
+ _b_fraction * _b_denominator / 1024**3,
+ _b_denominator / 1024**3,
+ )
+ except Exception as _budget_err:
+ logger.warning("Could not apply the training VRAM budget: %s", _budget_err)
+
if config.get("is_decision", False):
try:
_download_decision_checkpoint(event_queue, config)
@@ -4467,6 +4567,9 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
+ offload_layers = config.get("offload_layers", 0),
+ prefetch_depth = config.get("prefetch_depth", 2),
+ offload_plan_shape = _offload_plan_shape(config),
)
fallback_error = (
_model_cache_fallback_error(config, trainer.model_load_error)
@@ -4533,6 +4636,9 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
actual_model_repo_id = config.get("actual_model_repo_id"),
model_revision = model_revision,
use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
+ offload_layers = config.get("offload_layers", 0),
+ prefetch_depth = config.get("prefetch_depth", 2),
+ offload_plan_shape = _offload_plan_shape(config),
)
finally:
_load_watchdog_stop.set()
@@ -4541,7 +4647,9 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
- error_msg = trainer.training_progress.error or "Failed to load model"
+ error_msg = _with_vram_budget_hint(
+ config, trainer.training_progress.error or "Failed to load model"
+ )
event_queue.put(
{
"type": "error",
@@ -4635,7 +4743,9 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
event_queue.put(
{
"type": "error",
- "error": trainer.training_progress.error or "Failed to prepare model",
+ "error": _with_vram_budget_hint(
+ config, trainer.training_progress.error or "Failed to prepare model"
+ ),
"stack": "",
"ts": time.time(),
}
@@ -4786,7 +4896,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
event_queue.put(
{
"type": "error",
- "error": _oom_msg,
+ "error": _with_vram_budget_hint(config, _oom_msg),
"stack": traceback.format_exc(limit = 20),
"ts": time.time(),
}
@@ -4954,6 +5064,7 @@ def _create_trainer_progress_callback(event_queue: Any) -> Callable[[TrainingPro
"num_tokens": progress.num_tokens,
"eval_loss": progress.eval_loss,
"status_message": progress.status_message,
+ "offload": getattr(progress, "offload", None),
"ts": time.time(),
}
)
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 4bf17b78e6c..5c22722269d 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -7,7 +7,7 @@ import math
import re
from pathlib import Path, PureWindowsPath
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-from typing import Any, Optional, List, Dict, Literal, Union
+from typing import Annotated, Any, Optional, List, Dict, Literal, Union
from hub.schemas.inventory import ModelFormat
from utils.hf_dataset_options import (
@@ -113,6 +113,10 @@ def _resolve_inventory_handle(value: str) -> str:
return resolve_inventory_handle(value)
+# The strings trainer.normalize_gradient_checkpointing turns into False.
+_CHECKPOINTING_OFF = ("false", "0", "no", "none", "off")
+
+
class TrainingStartRequest(BaseModel):
"""Request schema for starting training"""
@@ -571,6 +575,21 @@ class TrainingStartRequest(BaseModel):
lora_dropout: float = Field(0.0, description = "LoRA dropout")
target_modules: List[str] = Field(default_factory = list, description = "Target modules for LoRA")
gradient_checkpointing: str = Field("", description = "Gradient checkpointing setting")
+ offload_layers: Union[Literal["auto"], Annotated[int, Field(ge = 0, le = 1024)]] = Field(
+ 0,
+ description = "Decoder layers kept in host RAM and streamed to the GPU during training "
+ "(0 = off, 'auto' = as few as fit)",
+ )
+ offload_vram_gb: Optional[float] = Field(
+ None,
+ gt = 0,
+ le = 4096,
+ description = "VRAM this run may use, in GiB; offload_layers = 'auto' sizes to it",
+ )
+ prefetch_depth: Union[Literal["auto"], Annotated[int, Field(ge = 1, le = 8)]] = Field(
+ 2,
+ description = "Offloaded layers fetched ahead of the one running ('auto' = measured)",
+ )
use_rslora: bool = Field(False, description = "Use RSLoRA")
use_loftq: bool = Field(False, description = "Use LoftQ")
use_dora: bool = Field(False, description = "Use DoRA")
@@ -657,6 +676,20 @@ class TrainingStartRequest(BaseModel):
raise ValueError("Either num_epochs or max_steps must be > 0; both cannot be 0.")
return self
+ @model_validator(mode = "after")
+ def _check_offload_has_checkpointing(self) -> "TrainingStartRequest":
+ # install_block_swap refuses swapped layers without checkpointing; say so before loading.
+ if (
+ self.offload_layers
+ and self.training_type != "Full Finetuning"
+ and self.gradient_checkpointing.strip().lower() in _CHECKPOINTING_OFF
+ ):
+ raise ValueError(
+ "offload_layers needs gradient checkpointing: set gradient_checkpointing to "
+ "'unsloth' or 'true', or offload_layers to 0."
+ )
+ return self
+
@model_validator(mode = "after")
def _validate_lora_variant_flags(self) -> "TrainingStartRequest":
# A direct API, YAML or CLI caller can bypass the frontend; nothing downstream breaks, but reject
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 323a853f31c..1a536959500 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -1509,6 +1509,18 @@ async def get_hardware_utilization(current_subject: str = Depends(get_current_su
return await asyncio.to_thread(get_gpu_utilization)
[email protected]("/offload")
+async def get_offload_state(current_subject: str = Depends(get_current_subject)):
+ """Offload layers state from the last logged step: which decoder layers sit on the GPU or in
+ host RAM, prefetch depth, copy / stall / compute time, VRAM. ``{"active": false}`` when the run
+ does not offload. Polled by the live training view."""
+ progress = get_training_backend().trainer.get_training_progress()
+ offload = getattr(progress, "offload", None)
+ if not offload:
+ return {"active": False}
+ return {"active": True, **offload}
+
+
@router.get("/hardware/visible")
async def get_visible_hardware_utilization(current_subject: str = Depends(get_current_subject)):
from utils.hardware import get_visible_gpu_utilization, gpu_query
@@ -1973,6 +1985,9 @@ async def start_training(
"gradient_checkpointing": request.gradient_checkpointing.strip()
if request.gradient_checkpointing and request.gradient_checkpointing.strip()
else "unsloth",
+ "offload_layers": request.offload_layers,
+ "offload_vram_gb": request.offload_vram_gb,
+ "prefetch_depth": request.prefetch_depth,
"use_rslora": request.use_rslora,
"use_loftq": request.use_loftq,
"use_dora": request.use_dora,
diff --git a/studio/backend/tests/test_audio_lora_logging.py b/studio/backend/tests/test_audio_lora_logging.py
index e57c39bb844..13cc9a62472 100644
--- a/studio/backend/tests/test_audio_lora_logging.py
+++ b/studio/backend/tests/test_audio_lora_logging.py
@@ -59,6 +59,7 @@ def test_audio_lora_setup_accepts_real_structlog(monkeypatch, audio_type):
should_stop = False,
_update_progress = progress,
_use_gradient_checkpointing = "unsloth",
+ _offload_peft_kwargs = lambda: {},
)
assert namespace["prepare_model_for_training"](instance) is True
diff --git a/studio/backend/tests/test_training_offload_layers.py b/studio/backend/tests/test_training_offload_layers.py
new file mode 100644
index 00000000000..70362db17dd
--- /dev/null
+++ b/studio/backend/tests/test_training_offload_layers.py
@@ -0,0 +1,319 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Offload layers in Unsloth Studio training: the request fields, the VRAM budget, the kwargs each
+load and LoRA path receives, and the live snapshot the training view polls."""
+
+from __future__ import annotations
+
+import sys
+from types import SimpleNamespace
+
+import pytest
+from pydantic import ValidationError
+
+from core.training.training import _build_training_worker_config
+from core.training.worker import _training_vram_budget_fraction, _with_vram_budget_hint
+from models.training import TrainingStartRequest
+
+GIB = 1024**3
+
+
+def _request(**kwargs) -> TrainingStartRequest:
+ return TrainingStartRequest(
+ model_name = "unsloth/Qwen3-0.6B", training_type = "LoRA/QLoRA", format_type = "alpaca", **kwargs
+ )
+
+
+def test_offload_fields_default_to_off():
+ r = _request()
+ assert r.offload_layers == 0 and r.offload_vram_gb is None and r.prefetch_depth == 2
+
+
[email protected]("value", [0, 14, "auto"])
+def test_offload_layers_takes_a_count_or_auto(value):
+ assert _request(offload_layers = value).offload_layers == value
+
+
[email protected]("value", [-1, 2000, "max", 1.5])
+def test_offload_layers_refuses_what_core_cannot_use(value):
+ with pytest.raises(ValidationError):
+ _request(offload_layers = value)
+
+
[email protected]("value", [0, 9, "fast"])
+def test_prefetch_depth_refuses_out_of_range(value):
+ with pytest.raises(ValidationError):
+ _request(prefetch_depth = value)
+
+
+def test_vram_budget_must_be_positive():
+ assert _request(offload_vram_gb = 11.5).offload_vram_gb == 11.5
+ with pytest.raises(ValidationError):
+ _request(offload_vram_gb = 0)
+
+
+def test_worker_config_carries_the_offload_fields():
+ cfg = _build_training_worker_config(
+ {
+ "model_name": "org/model",
+ "offload_layers": "auto",
+ "offload_vram_gb": 12.0,
+ "prefetch_depth": "auto",
+ }
+ )
+ assert (cfg["offload_layers"], cfg["offload_vram_gb"], cfg["prefetch_depth"]) == (
+ "auto",
+ 12.0,
+ "auto",
+ )
+ off = _build_training_worker_config({"model_name": "org/model"})
+ assert (off["offload_layers"], off["offload_vram_gb"], off["prefetch_depth"]) == (0, None, 2)
+
+
+def test_budget_fraction_holds_the_run_to_its_gib():
+ assert _training_vram_budget_fraction(8, 32 * GIB) == pytest.approx(0.25)
+ # Never looser than the OOM guard's cap.
+ assert _training_vram_budget_fraction(31, 32 * GIB, current = 0.8) == pytest.approx(0.8)
+ assert _training_vram_budget_fraction(64, 32 * GIB) == pytest.approx(1.0)
+ for budget in (None, 0, -2, "x", float("nan")):
+ assert _training_vram_budget_fraction(budget, 32 * GIB) is None
+ assert _training_vram_budget_fraction(8, 0) is None
+
+
+def _trainer(
+ offload_layers = 0,
+ prefetch_depth = 2,
+ model = None,
+):
+ from core.training.trainer import UnslothTrainer
+
+ t = UnslothTrainer.__new__(UnslothTrainer)
+ t._offload_layers, t._prefetch_depth, t.model = offload_layers, prefetch_depth, model
+ t._offload_plan_shape = {}
+ return t
+
+
+def test_load_and_lora_kwargs_follow_the_setting():
+ assert _trainer()._offload_load_kwargs() == {} and _trainer()._offload_peft_kwargs() == {}
+ t = _trainer("auto", "auto")
+ assert t._offload_load_kwargs() == {
+ "offload_layers": "auto",
+ "device_map_planner_kwargs": {"prefetch_depth": "auto"},
+ }
+ assert t._offload_peft_kwargs() == {"offload_layers": "auto", "prefetch_depth": "auto"}
+
+
+def test_a_fixed_count_on_several_gpus_waits_for_peft():
+ # Core refuses a count at load on a multi-GPU map; get_peft_model installs it there.
+ for device_map in ("unsloth_balanced", "balanced"):
+ assert _trainer(8, 2)._offload_load_kwargs(device_map) == {}
+ assert _trainer("auto", 2)._offload_load_kwargs(device_map)["offload_layers"] == "auto"
+ assert _trainer(8, 2)._offload_peft_kwargs() == {"offload_layers": 8, "prefetch_depth": 2}
+ assert _trainer(8, 2)._offload_load_kwargs("sequential")["offload_layers"] == 8
+
+
+def test_every_offload_capable_load_path_passes_the_kwargs():
+ import inspect
+ from core.training import trainer
+
+ src = inspect.getsource(trainer.UnslothTrainer.load_model)
+ # Text, Orpheus (snac), audio VLM and vision; the codec / Whisper paths have no decoder stack to stream.
+ assert src.count("**self._offload_load_kwargs(device_map)") == 4
+ peft = inspect.getsource(trainer.UnslothTrainer.prepare_model_for_training)
+ assert peft.count("**self._offload_peft_kwargs()") == 3
+
+
+def test_snapshot_is_none_without_a_swapper():
+ assert _trainer(model = SimpleNamespace())._offload_snapshot() is None
+ assert _trainer(model = None)._offload_snapshot() is None
+ # An unsloth_zoo without stats() still trains, just without the panel.
+ swapper = SimpleNamespace(indices = [1])
+ assert _trainer(model = SimpleNamespace(_unsloth_block_swap = swapper))._offload_snapshot() is None
+
+
+def test_snapshot_keys_layers_as_strings(monkeypatch):
+ from core.training import trainer
+
+ monkeypatch.setattr(trainer.torch.cuda, "is_available", lambda: False)
+ swapper = SimpleNamespace(
+ stats = lambda: {"state": {14: "host", 15: "gpu"}, "swapped": [14, 15], "prefetch_depth": 2}
+ )
+ snap = _trainer(model = SimpleNamespace(_unsloth_block_swap = swapper))._offload_snapshot()
+ assert snap["state"] == {"14": "host", "15": "gpu"} and snap["prefetch_depth"] == 2
+
+
+def test_offload_route_reports_inactive_until_a_step_carries_stats(monkeypatch):
+ import asyncio
+ from routes import training as route
+
+ progress = SimpleNamespace(offload = None)
+ backend = SimpleNamespace(trainer = SimpleNamespace(get_training_progress = lambda: progress))
+ monkeypatch.setattr(route, "get_training_backend", lambda: backend)
+ assert asyncio.run(route.get_offload_state(current_subject = "u")) == {"active": False}
+ progress.offload = {"swapped": [3], "prefetch_depth": 2}
+ assert asyncio.run(route.get_offload_state(current_subject = "u")) == {
+ "active": True,
+ "swapped": [3],
+ "prefetch_depth": 2,
+ }
+
+
+def test_worker_progress_events_carry_the_snapshot():
+ import inspect
+ from core.training import worker
+ assert '"offload": getattr(progress, "offload", None)' in inspect.getsource(
+ worker._create_trainer_progress_callback
+ )
+
+
+def test_backend_keeps_the_last_offload_snapshot():
+ from core.training.training import TrainingBackend
+
+ backend = TrainingBackend()
+ snap = {"swapped": [14, 15], "prefetch_depth": 2}
+ backend._handle_event({"type": "progress", "step": 1, "loss": 1.0, "offload": snap})
+ # A later step without stats must not blank the panel.
+ backend._handle_event({"type": "progress", "step": 2, "loss": 0.9})
+ assert backend._progress.offload == snap
+
+
+# The planner's refusal for Qwen3-0.6B at a 1 GiB budget (unsloth_zoo DeviceMapInfeasible).
+_INFEASIBLE = (
+ "Even with 27 of 28 decoder layers in host RAM the model does not fit while keeping 0.78 GiB "
+ "free for training on cuda:0 (0.96 GiB free). Lower max_seq_length or the batch size, or add a GPU."
+)
+
+
+ "message", [_INFEASIBLE, "GPU ran out of VRAM during training.", "CUDA out of memory."]
+)
+def test_a_run_that_does_not_fit_names_its_vram_budget(message):
+ hinted = _with_vram_budget_hint({"offload_vram_gb": 1.5}, message)
+ assert hinted.startswith(message) and "1.5 GiB VRAM budget" in hinted
+
+
+def test_budget_hint_stays_out_of_unrelated_or_unbudgeted_errors():
+ assert _with_vram_budget_hint({"offload_vram_gb": 4}, "Access denied") == "Access denied"
+ assert _with_vram_budget_hint({}, _INFEASIBLE) == _INFEASIBLE
+
+
+def test_every_worker_failure_path_carries_the_budget_hint():
+ from pathlib import Path
+ src = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(
+ encoding = "utf-8"
+ )
+ # Load, LoRA prepare (offload_layers = "auto" plans in get_peft_model) and training OOM.
+ assert src.count("_with_vram_budget_hint(") == 4
+
+
[email protected]("checkpointing", ["none", "False", "off", "no", "0"])
[email protected]("offload", [14, "auto"])
+def test_offload_without_checkpointing_is_refused_up_front(checkpointing, offload):
+ with pytest.raises(ValidationError, match = "needs gradient checkpointing"):
+ _request(offload_layers = offload, gradient_checkpointing = checkpointing)
+
+
+def test_offload_with_checkpointing_or_off_is_accepted():
+ _request(offload_layers = "auto", gradient_checkpointing = "unsloth")
+ _request(offload_layers = 0, gradient_checkpointing = "none")
+
+
+def test_worker_applies_the_budget_only_where_auto_offload_sizes_to_it():
+ from pathlib import Path
+
+ src = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(
+ encoding = "utf-8"
+ )
+ gate = src[src.index("# ── 2b. Training VRAM budget ──") :][:900]
+ for needle in (
+ 'config.get("offload_layers") == "auto"',
+ 'config.get("is_decision")',
+ 'config.get("is_embedding")',
+ 'config.get("training_type", "LoRA/QLoRA") in ("LoRA/QLoRA", "Continued Pretraining")',
+ ):
+ assert needle in gate
+ # The budget lands before the decision / embedding branches return, so the gate has to.
+ assert src.index("# ── 2b. Training VRAM budget ──") < src.index(
+ 'if config.get("is_decision", False):'
+ )
+
+
+def test_multi_gpu_vision_and_audio_runs_skip_the_budget():
+ # Core cannot offload a vision / audio load spread over GPUs, and LoRA setup allocates before it swaps,
+ # so a cap there would OOM; the budget is skipped for those runs, and the OOM hint must not name it.
+ from pathlib import Path
+
+ src = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(
+ encoding = "utf-8"
+ )
+ gate = src[src.index("_budget_unsupported = ") :][:300]
+ for needle in (
+ "(len(gpu_ids) if gpu_ids else _visible_gpu_count()) > 1",
+ 'config.get("is_dataset_image")',
+ 'config.get("is_dataset_audio")',
+ ):
+ assert needle in gate
+ skip = src.index("if _budget_unsupported:")
+ assert 'config["offload_vram_gb"] = None' in src[skip : src.index("elif _wants_budget:")]
+ assert (
+ skip
+ < src.index("elif _wants_budget:")
+ < src.index("set_per_process_memory_fraction(_b_fraction")
+ )
+
+
+def test_an_inherited_mask_counts_its_visible_gpus(monkeypatch):
+ # UUID / MIG masks resolve no ids, yet the load still spreads over every visible card.
+ from core.training import worker
+
+ cuda = SimpleNamespace(is_available = lambda: True, device_count = lambda: 2)
+ monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda = cuda))
+ assert worker._visible_gpu_count() == 2
+ cuda.is_available = lambda: False
+ assert worker._visible_gpu_count() == 0
+
+
+def test_disabled_checkpointing_aliases_match_the_trainer():
+ import ast
+ from pathlib import Path
+
+ from models.training import _CHECKPOINTING_OFF
+
+ src = (Path(__file__).resolve().parents[1] / "core" / "training" / "trainer.py").read_text(
+ encoding = "utf-8"
+ )
+ fn = next(
+ n
+ for n in ast.parse(src).body
+ if isinstance(n, ast.FunctionDef) and n.name == "normalize_gradient_checkpointing"
+ )
+ off = next(
+ ast.literal_eval(node.test.comparators[0])
+ for node in ast.walk(fn)
+ if isinstance(node, ast.If)
+ and isinstance(node.test, ast.Compare)
+ and isinstance(node.body[0], ast.Return)
+ and getattr(node.body[0].value, "value", None) is False
+ )
+ assert set(off) == set(_CHECKPOINTING_OFF)
+
+
+def test_auto_plans_at_load_for_the_runs_batch_and_rank():
+ from core.training.worker import _offload_plan_shape
+
+ trainer = _trainer("auto")
+ trainer._offload_plan_shape = _offload_plan_shape({"batch_size": 4, "lora_r": 64})
+ assert trainer._offload_load_kwargs()["device_map_planner_kwargs"] == {
+ "prefetch_depth": 2,
+ "batch_size": 4,
+ "lora_rank": 64,
+ }
+
+
+def test_every_worker_load_passes_the_plan_shape():
+ from pathlib import Path
+ src = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text(
+ encoding = "utf-8"
+ )
+ assert src.count("offload_plan_shape = _offload_plan_shape(config),") == 2
diff --git a/studio/frontend/src/config/training.ts b/studio/frontend/src/config/training.ts
index 397c7d986fc..5900c5b4a20 100644
--- a/studio/frontend/src/config/training.ts
+++ b/studio/frontend/src/config/training.ts
@@ -171,6 +171,9 @@ export const DEFAULT_HYPERPARAMS = {
packing: false,
trainOnCompletions: false,
gradientCheckpointing: "unsloth" as const,
+ offloadLayers: 0 as number | "auto",
+ offloadVramGb: null as number | null,
+ prefetchDepth: 2 as number | "auto",
randomSeed: 3407,
enableWandb: false,
wandbToken: "",
diff --git a/studio/frontend/src/features/studio/live-training-view.tsx b/studio/frontend/src/features/studio/live-training-view.tsx
index 92ce7d91615..2be7a6e2496 100644
--- a/studio/frontend/src/features/studio/live-training-view.tsx
+++ b/studio/frontend/src/features/studio/live-training-view.tsx
@@ -12,6 +12,7 @@ import type { ReactElement } from "react";
import { useEffect, useState } from "react";
import { useShallow } from "zustand/react/shallow";
import { ChartsSection } from "./sections/charts-section";
+import { OffloadPanel } from "./sections/offload-panel";
import { ProgressSection } from "./sections/progress-section";
import {
type RunConfigOverride,
@@ -195,6 +196,7 @@ export function LiveTrainingView(): ReactElement {
configOverride={runConfigOverride}
/>
</div>
+ <OffloadPanel isTrainingRunning={viewData.isTrainingRunning} />
<ChartsSection
currentStep={viewData.currentStep}
totalSteps={viewData.totalSteps}
diff --git a/studio/frontend/src/features/studio/sections/offload-panel-layout.ts b/studio/frontend/src/features/studio/sections/offload-panel-layout.ts
new file mode 100644
index 00000000000..00ee155a06b
--- /dev/null
+++ b/studio/frontend/src/features/studio/sections/offload-panel-layout.ts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+/** Placement of each layer while the window sits at `pos` in the offloaded list. */
+export function layerPlacement(
+ total: number,
+ swapped: number[],
+ depth: number,
+ pos: number,
+): ("resident" | "running" | "copying" | "host")[] {
+ const order = new Map(swapped.map((layer, k) => [layer, k]));
+ return Array.from({ length: total }, (_, layer) => {
+ const k = order.get(layer);
+ if (k === undefined) return "resident";
+ if (k === pos) return "running";
+ if (k > pos && k <= pos + depth) return "copying";
+ return "host";
+ });
+}
+
+/** The layer count a Count box edit sets, or null to keep the current one (empty, zero, not a number). */
+export function offloadCountFromInput(text: string): number | null {
+ const count = Math.floor(Number(text));
+ return count > 0 ? count : null;
+}
diff --git a/studio/frontend/src/features/studio/sections/offload-panel.tsx b/studio/frontend/src/features/studio/sections/offload-panel.tsx
new file mode 100644
index 00000000000..5c9dd2fd51a
--- /dev/null
+++ b/studio/frontend/src/features/studio/sections/offload-panel.tsx
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { SectionCard } from "@/components/section-card";
+import { authFetch } from "@/features/auth";
+import { useT } from "@/i18n";
+import { cn } from "@/lib/utils";
+import { RamMemoryIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import type { ReactElement } from "react";
+import { useEffect, useState } from "react";
+import { layerPlacement } from "./offload-panel-layout";
+
+/** `GET /api/train/offload`: unsloth_zoo BlockSwap.stats() from the last logged step. */
+export interface OffloadState {
+ active: boolean;
+ total_layers?: number;
+ swapped?: number[];
+ state?: Record<string, "gpu" | "host" | "copying" | "held">;
+ prefetch_depth?: number;
+ auto_depth?: boolean;
+ depth_settled?: boolean;
+ host_bytes?: number;
+ pinned_bytes?: number;
+ pool_bytes?: number;
+ timing?: boolean;
+ copies?: number;
+ copy_ms?: number;
+ stall_ms?: number;
+ compute_ms?: number;
+ layers?: number;
+ vram_allocated_bytes?: number;
+ vram_peak_bytes?: number;
+ vram_total_bytes?: number;
+ vram_fraction?: number;
+}
+
+const GIB = 1024 ** 3;
+// One sweep of the window takes about this long on screen; real layers run in milliseconds.
+const SWEEP_MS = 4000;
+
+export function useOffloadState(enabled: boolean, intervalMs = 2000): OffloadState {
+ const [data, setData] = useState<OffloadState>({ active: false });
+ useEffect(() => {
+ if (!enabled) {
+ setData({ active: false });
+ return;
+ }
+ let cancelled = false;
+ async function poll() {
+ try {
+ const res = await authFetch("/api/train/offload");
+ if (!res.ok || cancelled) return;
+ const json = (await res.json()) as OffloadState;
+ if (!cancelled) setData(json);
+ } catch {
+ // Retry on the next poll.
+ }
+ }
+ void poll();
+ const timer = setInterval(() => void poll(), intervalMs);
+ return () => {
+ cancelled = true;
+ clearInterval(timer);
+ };
+ }, [enabled, intervalMs]);
+ return data;
+}
+
+function useSweep(count: number): number {
+ const [pos, setPos] = useState(0);
+ useEffect(() => {
+ if (count <= 0) return;
+ let frame = 0;
+ const start = performance.now();
+ const tick = (now: number) => {
+ setPos(Math.floor(((now - start) % SWEEP_MS) / (SWEEP_MS / count)));
+ frame = requestAnimationFrame(tick);
+ };
+ frame = requestAnimationFrame(tick);
+ return () => cancelAnimationFrame(frame);
+ }, [count]);
+ return pos;
+}
+
+const CELL = {
+ resident: "bg-control-accent/70",
+ running: "bg-control-accent ring-2 ring-control-accent/40",
+ copying: "bg-amber-500/80",
+ host: "border border-border bg-transparent",
+} as const;
+
+function Stat({ label, value }: { label: string; value: string }): ReactElement {
+ return (
+ <div className="flex flex-col gap-0.5 rounded-xl bg-muted/50 px-3 py-2">
+ <span className="text-ui-11 text-muted-foreground">{label}</span>
+ <span className="font-mono text-sm">{value}</span>
+ </div>
+ );
+}
+
+export function OffloadPanel({ isTrainingRunning }: { isTrainingRunning: boolean }): ReactElement | null {
+ const t = useT();
+ const s = useOffloadState(isTrainingRunning);
+ const swapped = s.swapped ?? [];
+ const total = s.total_layers ?? 0;
+ const depth = s.prefetch_depth ?? 2;
+ const pos = useSweep(swapped.length);
+ if (!s.active || total === 0) return null;
+
+ const cells = layerPlacement(total, swapped, depth, pos);
+ const perCopy = s.copies ? (s.copy_ms ?? 0) / s.copies : null;
+ const perLayer = s.layers ? (s.compute_ms ?? 0) / s.layers : null;
+ const busy = (s.compute_ms ?? 0) + (s.stall_ms ?? 0);
+ const waiting = busy > 0 ? (100 * (s.stall_ms ?? 0)) / busy : null;
+ const vramTotal = s.vram_total_bytes ?? 0;
+ const used = s.vram_peak_bytes ?? s.vram_allocated_bytes ?? 0;
+ const budget = s.vram_fraction && s.vram_fraction < 1 ? s.vram_fraction * vramTotal : null;
+ const ms = (v: number | null) => (v == null ? "--" : `${v.toFixed(1)} ms`);
+
+ return (
+ <SectionCard
+ icon={<HugeiconsIcon icon={RamMemoryIcon} className="size-5" />}
+ title={t("studio.params.offloadPanelTitle")}
+ description={`${swapped.length} / ${total} ${t("studio.params.offloadPanelSwapped")} · ${t(
+ "studio.params.offloadPanelDepth",
+ )} ${depth}${s.auto_depth ? ` (${t("studio.params.offloadAuto")})` : ""}`}
+ className="shadow-border border border-border/60 bg-card/90 ring-0 backdrop-blur-sm"
+ >
+ <div className="flex flex-wrap items-start gap-6">
+ <div className="grid grid-cols-8 gap-1" aria-label={t("studio.params.offloadPanelTitle")}>
+ {cells.map((cell, layer) => (
+ <div
+ // biome-ignore lint/suspicious/noArrayIndexKey: one cell per decoder layer, fixed order
+ key={layer}
+ title={`${layer}`}
+ className={cn("size-4 rounded-[3px] transition-colors", CELL[cell])}
+ />
+ ))}
+ </div>
+ <div className="flex min-w-48 flex-1 flex-col gap-3">
+ <div className="flex flex-wrap gap-3 text-ui-11 text-muted-foreground">
+ <span className="flex items-center gap-1.5">
+ <span className={cn("size-2.5 rounded-[2px]", CELL.resident)} />
+ {t("studio.params.offloadPanelGpu")}
+ </span>
+ <span className="flex items-center gap-1.5">
+ <span className={cn("size-2.5 rounded-[2px]", CELL.copying)} />
+ {t("studio.params.offloadPanelCopying")}
+ </span>
+ <span className="flex items-center gap-1.5">
+ <span className={cn("size-2.5 rounded-[2px]", CELL.host)} />
+ {t("studio.params.offloadPanelHost")}
+ </span>
+ </div>
+ {vramTotal > 0 && (
+ <div className="flex flex-col gap-1">
+ <div className="flex justify-between text-ui-11 text-muted-foreground">
+ <span>{t("studio.params.offloadPanelVram")}</span>
+ <span className="font-mono">
+ {(used / GIB).toFixed(1)} / {((budget ?? vramTotal) / GIB).toFixed(1)} GiB
+ </span>
+ </div>
+ <div className="relative h-2 overflow-hidden rounded-full bg-muted">
+ <div
+ className="h-full rounded-full bg-control-accent"
+ style={{ width: `${Math.min(100, (100 * used) / vramTotal)}%` }}
+ />
+ {budget != null && (
+ <div
+ className="absolute inset-y-0 w-0.5 bg-[color-mix(in_oklab,var(--foreground)_calc(70%*var(--contrast-wash-gain,1)),transparent)]"
+ style={{ left: `${(100 * budget) / vramTotal}%` }}
+ />
+ )}
+ </div>
+ </div>
+ )}
+ <div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
+ <Stat label={t("studio.params.offloadPanelCopy")} value={ms(perCopy)} />
+ <Stat label={t("studio.params.offloadPanelCompute")} value={ms(perLayer)} />
+ <Stat
+ label={t("studio.params.offloadPanelStall")}
+ value={waiting == null ? "--" : `${waiting.toFixed(1)}%`}
+ />
+ <Stat
+ label={t("studio.params.offloadPanelPinned")}
+ value={`${((s.pinned_bytes ?? 0) / GIB).toFixed(1)} / ${((s.host_bytes ?? 0) / GIB).toFixed(1)} GiB`}
+ />
+ </div>
+ <p className="text-ui-11 text-muted-foreground">{t("studio.params.offloadPanelSweepNote")}</p>
+ </div>
+ </div>
+ </SectionCard>
+ );
+}
diff --git a/studio/frontend/src/features/studio/sections/training-memory-params.tsx b/studio/frontend/src/features/studio/sections/training-memory-params.tsx
index b15e794ed37..8e8513eeceb 100644
--- a/studio/frontend/src/features/studio/sections/training-memory-params.tsx
+++ b/studio/frontend/src/features/studio/sections/training-memory-params.tsx
@@ -27,6 +27,9 @@ import { HugeiconsIcon } from "@hugeicons/react";
import type { ReactElement } from "react";
import { useShallow } from "zustand/react/shallow";
import { ParamsRow } from "./params-section-controls";
+import { OffloadLayersParams } from "./training-offload-params";
+import { offloadHardwareSupported, offloadSupported } from "@/features/training/api/mappers";
+import { useSystemInfo } from "@/hooks/use-system";
const VISION_IMAGE_SIZE_PRESETS = [256, 384, 512, 768, 1024, 1536, 2048];
@@ -114,6 +117,7 @@ function TrainOnCompletionsOption({
export function TrainingMemoryParams(): ReactElement {
const t = useT();
const isMac = usePlatformStore((state) => state.deviceType === "mac");
+ const system = useSystemInfo();
const store = useTrainingConfigStore(
useShallow((state) => ({
selectedModel: state.selectedModel,
@@ -125,7 +129,9 @@ export function TrainingMemoryParams(): ReactElement {
gradientCheckpointing: state.gradientCheckpointing,
isVisionModel: state.isVisionModel,
isEmbeddingModel: state.isEmbeddingModel,
+ isAudioModel: state.isAudioModel,
isDecision: state.modelType === "decision",
+ modelType: state.modelType,
isDatasetImage: state.isDatasetImage,
visionImageSize: state.visionImageSize,
setVisionImageSize: state.setVisionImageSize,
@@ -243,6 +249,9 @@ export function TrainingMemoryParams(): ReactElement {
</SelectContent>
</Select>
</ParamsRow>
+ {!isMac && offloadHardwareSupported(system) && offloadSupported(store) && (
+ <OffloadLayersParams />
+ )}
{showPacking && (
<PackingOption
isMac={isMac}
diff --git a/studio/frontend/src/features/studio/sections/training-offload-params.tsx b/studio/frontend/src/features/studio/sections/training-offload-params.tsx
new file mode 100644
index 00000000000..bd19db8d1bf
--- /dev/null
+++ b/studio/frontend/src/features/studio/sections/training-offload-params.tsx
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Input } from "@/components/ui/input";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { useTrainingConfigStore } from "@/features/training";
+import type { OffloadLayers, PrefetchDepth } from "@/features/training/types/config";
+import { useT } from "@/i18n";
+import { type ReactElement, useState } from "react";
+import { useShallow } from "zustand/react/shallow";
+import { offloadCountFromInput } from "./offload-panel-layout";
+import { ParamsRow } from "./params-section-controls";
+
+type OffloadMode = "off" | "auto" | "layers";
+
+function modeOf(value: OffloadLayers): OffloadMode {
+ if (value === "auto") return "auto";
+ return value > 0 ? "layers" : "off";
+}
+
+const PREFETCH_CHOICES: PrefetchDepth[] = ["auto", 1, 2, 3, 4];
+
+export function OffloadLayersParams(): ReactElement {
+ const t = useT();
+ const store = useTrainingConfigStore(
+ useShallow((state) => ({
+ offloadLayers: state.offloadLayers,
+ offloadVramGb: state.offloadVramGb,
+ prefetchDepth: state.prefetchDepth,
+ setOffloadLayers: state.setOffloadLayers,
+ setOffloadVramGb: state.setOffloadVramGb,
+ setPrefetchDepth: state.setPrefetchDepth,
+ })),
+ );
+ // The count box's text while it is being edited: clearing it to retype must not read as Off.
+ const [countDraft, setCountDraft] = useState<string | null>(null);
+ const mode = countDraft !== null ? "layers" : modeOf(store.offloadLayers);
+
+ return (
+ <>
+ <ParamsRow
+ label={t("studio.params.offloadLayers")}
+ tooltip={t("studio.params.offloadLayersTooltip")}
+ >
+ <div className="flex items-center gap-2">
+ <Select
+ value={mode}
+ onValueChange={(value) => {
+ const next = value as OffloadMode;
+ setCountDraft(null);
+ if (next === "off") store.setOffloadLayers(0);
+ else if (next === "auto") store.setOffloadLayers("auto");
+ else store.setOffloadLayers(typeof store.offloadLayers === "number" && store.offloadLayers > 0 ? store.offloadLayers : 8);
+ }}
+ >
+ <SelectTrigger className="w-28">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="off">{t("studio.params.offloadOff")}</SelectItem>
+ <SelectItem value="auto">{t("studio.params.offloadAuto")}</SelectItem>
+ <SelectItem value="layers">{t("studio.params.offloadCount")}</SelectItem>
+ </SelectContent>
+ </Select>
+ {mode === "layers" && (
+ <Input
+ type="number"
+ inputMode="numeric"
+ min={1}
+ step={1}
+ aria-label={t("studio.params.offloadCount")}
+ value={countDraft ?? (store.offloadLayers === "auto" ? "" : store.offloadLayers || "")}
+ onChange={(e) => {
+ setCountDraft(e.target.value);
+ const count = offloadCountFromInput(e.target.value);
+ if (count !== null) store.setOffloadLayers(count);
+ }}
+ // An empty or zero box keeps the last count; Off is chosen from the select.
+ onBlur={() => setCountDraft(null)}
+ className="w-20 font-mono"
+ />
+ )}
+ </div>
+ </ParamsRow>
+ {mode === "auto" && (
+ <ParamsRow
+ label={t("studio.params.offloadVramBudget")}
+ tooltip={t("studio.params.offloadVramBudgetTooltip")}
+ >
+ <Input
+ type="number"
+ inputMode="decimal"
+ min={1}
+ step={0.5}
+ placeholder={t("studio.params.offloadWholeCard")}
+ title={t("studio.params.offloadWholeCard")}
+ value={store.offloadVramGb ?? ""}
+ onChange={(e) => {
+ const gb = Number(e.target.value);
+ store.setOffloadVramGb(e.target.value === "" || !(gb > 0) ? null : gb);
+ }}
+ className="w-36 font-mono placeholder:font-sans"
+ />
+ </ParamsRow>
+ )}
+ {mode !== "off" && (
+ <ParamsRow
+ label={t("studio.params.prefetchDepth")}
+ tooltip={t("studio.params.prefetchDepthTooltip")}
+ >
+ <Select
+ value={String(store.prefetchDepth)}
+ onValueChange={(value) =>
+ store.setPrefetchDepth(value === "auto" ? "auto" : Number(value))
+ }
+ >
+ <SelectTrigger className="w-28">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {PREFETCH_CHOICES.map((choice) => (
+ <SelectItem key={String(choice)} value={String(choice)}>
+ {choice === "auto" ? t("studio.params.offloadAuto") : choice}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </ParamsRow>
+ )}
+ </>
+ );
+}
diff --git a/studio/frontend/src/features/training/api/mappers.ts b/studio/frontend/src/features/training/api/mappers.ts
index 1687bc06e7c..71576c1cc4f 100644
--- a/studio/frontend/src/features/training/api/mappers.ts
+++ b/studio/frontend/src/features/training/api/mappers.ts
@@ -5,6 +5,7 @@ import {
isRawTextDatasetFormat,
toBackendTrainingType,
} from "../lib/training-methods";
+import type { SystemInfoResponse } from "@/hooks/use-system";
import type { TrainingStartRequest } from "../types/api";
import type { TrainingConfigState } from "../types/config";
@@ -50,9 +51,76 @@ export function trainingLoadsIn4Bit(
return (adapterMethod && isQloraMethod) || (isCpt && isFourBitModel);
}
+/** Whether a run can offload layers: LoRA on the main trainer (embedding and decision models
+ * train elsewhere, and the Whisper / codec audio paths have no decoder stack to stream) with
+ * gradient checkpointing on, which swapped layers need. */
+export function offloadSupported(
+ config: Pick<
+ TrainingConfigState,
+ "trainingMethod" | "isEmbeddingModel" | "isAudioModel" | "modelType" | "gradientCheckpointing"
+ >,
+): boolean {
+ return (
+ config.trainingMethod !== "full" &&
+ !config.isEmbeddingModel &&
+ !config.isAudioModel &&
+ config.modelType !== "embeddings" &&
+ config.modelType !== "decision" &&
+ config.gradientCheckpointing !== "none"
+ );
+}
+
+/** Whether this host can swap layers to system RAM, by core's install_block_swap rule: a CUDA or
+ * ROCm backend with at least one card that is not a unified-memory APU (XPU, MLX and CPU have no
+ * swap path). True until `/api/system` answers; the backend refuses the same cases itself. */
+export function offloadHardwareSupported(
+ system: Pick<SystemInfoResponse, "status" | "device_backend" | "gpu"> | null | undefined,
+): boolean {
+ if (!system || system.status !== "ready") return true;
+ if (system.device_backend !== "cuda" && system.device_backend !== "rocm") return false;
+ const devices = system.gpu?.devices ?? [];
+ return devices.length === 0 || devices.some((device) => device.unified_memory !== true);
+}
+
+/** The offload fields, sent off wherever the controls are hidden. */
+export function offloadPayload(
+ config: Pick<
+ TrainingConfigState,
+ | "trainingMethod"
+ | "isEmbeddingModel"
+ | "isAudioModel"
+ | "modelType"
+ | "gradientCheckpointing"
+ | "offloadLayers"
+ | "offloadVramGb"
+ | "prefetchDepth"
+ >,
+ system: Parameters<typeof offloadHardwareSupported>[0] = null,
+): Pick<TrainingStartRequest, "offload_layers" | "offload_vram_gb" | "prefetch_depth"> {
+ const layers = !(offloadSupported(config) && offloadHardwareSupported(system))
+ ? 0
+ : config.offloadLayers === "auto"
+ ? "auto"
+ : Math.min(1024, Math.max(0, Math.floor(config.offloadLayers || 0)));
+ const depth =
+ config.prefetchDepth === "auto"
+ ? "auto"
+ : Math.min(8, Math.max(1, Math.floor(config.prefetchDepth || 2)));
+ return {
+ offload_layers: layers,
+ // The budget only sizes "auto"; with a fixed count or off it would cap the run for nothing.
+ offload_vram_gb:
+ layers === "auto" && config.offloadVramGb && config.offloadVramGb > 0
+ ? config.offloadVramGb
+ : null,
+ prefetch_depth: depth,
+ };
+}
+
export function buildTrainingStartPayload(
config: TrainingConfigState,
hfToken: string | null,
+ system: SystemInfoResponse | null = null,
): TrainingStartRequest {
const isDecision = config.modelType === "decision";
// Laya trains in 16-bit (LoRA or full); Clef and an LLM decision model also take QLoRA.
@@ -171,6 +239,7 @@ export function buildTrainingStartPayload(
lora_dropout: config.loraDropout,
target_modules: adapterMethod ? config.targetModules : [],
gradient_checkpointing: config.gradientCheckpointing,
+ ...offloadPayload(config, system),
use_rslora: loraVariants && config.loraVariant === "rslora",
use_loftq: loraVariants && config.loraVariant === "loftq",
use_dora: loraVariants && config.loraVariant === "dora",
diff --git a/studio/frontend/src/features/training/api/models-api.ts b/studio/frontend/src/features/training/api/models-api.ts
index 0cc9f1c5cf4..617f8d0b305 100644
--- a/studio/frontend/src/features/training/api/models-api.ts
+++ b/studio/frontend/src/features/training/api/models-api.ts
@@ -30,6 +30,9 @@ interface BackendTrainingDefaults {
eval_steps?: number;
weight_decay?: number;
random_seed?: number;
+ offload_layers?: number | "auto";
+ offload_vram_gb?: number | null;
+ prefetch_depth?: number | "auto";
vision_image_size?: number | string | null;
packing?: boolean;
train_on_completions?: boolean;
diff --git a/studio/frontend/src/features/training/lib/model-defaults-edit-policy.ts b/studio/frontend/src/features/training/lib/model-defaults-edit-policy.ts
index 247748df814..a89db45ab5e 100644
--- a/studio/frontend/src/features/training/lib/model-defaults-edit-policy.ts
+++ b/studio/frontend/src/features/training/lib/model-defaults-edit-policy.ts
@@ -25,6 +25,9 @@ export const MODEL_DEFAULT_STATE_KEYS = [
"trainOnCompletions",
"gradientCheckpointing",
"randomSeed",
+ "offloadLayers",
+ "offloadVramGb",
+ "prefetchDepth",
"visionImageSize",
"enableWandb",
"wandbProject",
diff --git a/studio/frontend/src/features/training/lib/model-defaults.ts b/studio/frontend/src/features/training/lib/model-defaults.ts
index 0463ac94f99..d9dd84aa71d 100644
--- a/studio/frontend/src/features/training/lib/model-defaults.ts
+++ b/studio/frontend/src/features/training/lib/model-defaults.ts
@@ -151,6 +151,21 @@ export function mapBackendModelConfigToTrainingPatch(
const randomSeed = toNumber(training?.random_seed);
if (randomSeed !== undefined) patch.randomSeed = randomSeed;
+ if (training?.offload_layers === "auto") patch.offloadLayers = "auto";
+ else {
+ const offloadLayers = toNumber(training?.offload_layers);
+ if (offloadLayers !== undefined) patch.offloadLayers = offloadLayers;
+ }
+ if (Object.hasOwn(training ?? {}, "offload_vram_gb")) {
+ const budget = toNumber(training?.offload_vram_gb);
+ patch.offloadVramGb = budget !== undefined && budget > 0 ? budget : null;
+ }
+ if (training?.prefetch_depth === "auto") patch.prefetchDepth = "auto";
+ else {
+ const depth = toNumber(training?.prefetch_depth);
+ if (depth !== undefined && depth >= 1) patch.prefetchDepth = Math.min(8, Math.floor(depth));
+ }
+
// Only patch when the config carries the key; model-switch reset lives in setSelectedModel.
if (Object.hasOwn(training ?? {}, "vision_image_size")) {
const raw = training?.vision_image_size;
diff --git a/studio/frontend/src/features/training/lib/start-fresh-training-run.ts b/studio/frontend/src/features/training/lib/start-fresh-training-run.ts
index 8585ee0212d..f0febe65591 100644
--- a/studio/frontend/src/features/training/lib/start-fresh-training-run.ts
+++ b/studio/frontend/src/features/training/lib/start-fresh-training-run.ts
@@ -10,6 +10,7 @@ import { translate } from "@/i18n";
import { primeNativeNotificationPermission } from "@/lib/native-notifications";
import { toast } from "@/lib/toast";
import { DatasetFormatError, checkDatasetFormat } from "../api/datasets-api";
+import { getCachedSystemInfo } from "@/hooks/use-system";
import { buildTrainingStartPayload } from "../api/mappers";
import {
TrainingStartError,
@@ -59,6 +60,7 @@ const ROLE_REMAP: Record<string, Record<string, string>> = {
type AttemptPhase = "preflight" | "transport" | "finished";
function captureTrainingStartInputs(config: TrainingConfigState) {
+ // No hardware info: /api/system answering mid-start is not a user edit.
return createTrainingStartInputIdentity(
buildTrainingStartPayload(config, null),
config,
@@ -547,7 +549,7 @@ async function submitFreshTrainingRun(
return attempt.cancel(translate(validation.errorKey));
}
- const payload = buildTrainingStartPayload(attempt.config, hfToken);
+ const payload = buildTrainingStartPayload(attempt.config, hfToken, getCachedSystemInfo());
if (!attempt.enterTransport()) {
return false;
}
diff --git a/studio/frontend/src/features/training/lib/yaml-config.ts b/studio/frontend/src/features/training/lib/yaml-config.ts
index 02c2b67b3ca..b137e7f534e 100644
--- a/studio/frontend/src/features/training/lib/yaml-config.ts
+++ b/studio/frontend/src/features/training/lib/yaml-config.ts
@@ -94,6 +94,9 @@ export function serializeConfigToYaml(
packing: state.packing,
train_on_completions: state.trainOnCompletions,
gradient_checkpointing: state.gradientCheckpointing,
+ offload_layers: state.offloadLayers,
+ offload_vram_gb: state.offloadVramGb,
+ prefetch_depth: state.prefetchDepth,
optim: state.optimizerType,
lr_scheduler_type: state.lrSchedulerType,
};
diff --git a/studio/frontend/src/features/training/stores/training-config-store.ts b/studio/frontend/src/features/training/stores/training-config-store.ts
index 0efc9284117..0c4a75b233b 100644
--- a/studio/frontend/src/features/training/stores/training-config-store.ts
+++ b/studio/frontend/src/features/training/stores/training-config-store.ts
@@ -1598,6 +1598,9 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
},
setGradientCheckpointing: (gradientCheckpointing) =>
setUserEdit({ gradientCheckpointing }),
+ setOffloadLayers: (offloadLayers) => setUserEdit({ offloadLayers }),
+ setOffloadVramGb: (offloadVramGb) => setUserEdit({ offloadVramGb }),
+ setPrefetchDepth: (prefetchDepth) => setUserEdit({ prefetchDepth }),
setRandomSeed: (randomSeed) => setUserEdit({ randomSeed }),
setEnableWandb: (enableWandb) => setUserEdit({ enableWandb }),
setWandbToken: (wandbToken) => setUserEdit({ wandbToken }),
diff --git a/studio/frontend/src/features/training/types/api.ts b/studio/frontend/src/features/training/types/api.ts
index 2781d21660d..55c279be86c 100644
--- a/studio/frontend/src/features/training/types/api.ts
+++ b/studio/frontend/src/features/training/types/api.ts
@@ -61,6 +61,9 @@ export interface TrainingStartRequest {
lora_dropout: number;
target_modules: string[];
gradient_checkpointing: string;
+ offload_layers: number | "auto";
+ offload_vram_gb: number | null;
+ prefetch_depth: number | "auto";
use_rslora: boolean;
use_loftq: boolean;
use_dora: boolean;
diff --git a/studio/frontend/src/features/training/types/config.ts b/studio/frontend/src/features/training/types/config.ts
index 24890545e91..2231a4fa3a1 100644
--- a/studio/frontend/src/features/training/types/config.ts
+++ b/studio/frontend/src/features/training/types/config.ts
@@ -63,6 +63,10 @@ export interface TrainingMethodProvenance {
/** Column-to-role mapping, e.g. { "problem": "user", "solution": "assistant", "context": "system" } */
export type DatasetManualMapping = Record<string, string>;
+/** Decoder layers kept in host RAM during LoRA training: a count (0 = off) or "auto". */
+export type OffloadLayers = number | "auto";
+export type PrefetchDepth = number | "auto";
+
export interface TrainingConfigState {
userEditRevision: number;
modelType: ModelType | null;
@@ -122,6 +126,9 @@ export interface TrainingConfigState {
packing: boolean;
trainOnCompletions: boolean;
gradientCheckpointing: GradientCheckpointing;
+ offloadLayers: OffloadLayers;
+ offloadVramGb: number | null;
+ prefetchDepth: PrefetchDepth;
randomSeed: number;
enableWandb: boolean;
wandbToken: string;
@@ -258,6 +265,9 @@ export interface TrainingConfigActions {
setPacking: (value: boolean) => void;
setTrainOnCompletions: (value: boolean) => void;
setGradientCheckpointing: (value: GradientCheckpointing) => void;
+ setOffloadLayers: (value: OffloadLayers) => void;
+ setOffloadVramGb: (value: number | null) => void;
+ setPrefetchDepth: (value: PrefetchDepth) => void;
setRandomSeed: (value: number) => void;
setEnableWandb: (value: boolean) => void;
setWandbToken: (value: string) => void;
diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts
index a290044de20..3f0665ff5b2 100644
--- a/studio/frontend/src/i18n/locales/ar.ts
+++ b/studio/frontend/src/i18n/locales/ar.ts
@@ -3394,6 +3394,28 @@ export const ar = {
"نسبة إجمالي خطوات التدريب بين عمليات التقييم (0-1). اضبطها على 0 لتعطيل التقييم. مثال: 0.01 = تقييم كل 1% من الخطوات.",
seed: "البذرة (Seed)",
seedTooltip: "بذرة عشوائية لإمكانية إعادة الإنتاج.",
+ offloadLayers: "تفريغ الطبقات",
+ offloadLayersTooltip: "يُبقي طبقات فك الترميز في ذاكرة النظام وينقل كل طبقة إلى وحدة معالجة الرسومات قبل تشغيلها مباشرة، فيمكن تدريب نموذج أكبر من ذاكرة الفيديو باستخدام LoRA. الوضع التلقائي يفرّغ أقل عدد يكفي، والعدد يفرّغ هذا العدد بالضبط. يتطلب نقاط التحقق للتدرج.",
+ offloadOff: "إيقاف",
+ offloadAuto: "تلقائي",
+ offloadCount: "عدد",
+ offloadVramBudget: "حد ذاكرة الفيديو (GiB)",
+ offloadVramBudgetTooltip: "أقصى ذاكرة فيديو يستخدمها هذا التدريب. الوضع التلقائي يفرّغ طبقات كافية للبقاء تحته، ويمكن لتدريبين مشاركة بطاقة واحدة بأخذ كل منهما جزءًا. اتركه فارغًا لاستخدام البطاقة كاملة.",
+ offloadWholeCard: "البطاقة كاملة",
+ prefetchDepth: "عمق الجلب المسبق",
+ prefetchDepthTooltip: "عدد الطبقات المفرّغة التي تُنسخ قبل الطبقة الجارية. الوضع التلقائي يبدأ من 1 ولا يحتفظ بعمق أكبر إلا إذا قلّل الانتظار فعلًا.",
+ offloadPanelTitle: "الطبقات المفرّغة",
+ offloadPanelGpu: "على وحدة الرسومات",
+ offloadPanelHost: "في ذاكرة النظام",
+ offloadPanelCopying: "قيد النسخ",
+ offloadPanelSwapped: "مفرّغة",
+ offloadPanelStall: "انتظار النسخ",
+ offloadPanelCopy: "النسخ لكل طبقة",
+ offloadPanelCompute: "الحساب لكل طبقة",
+ offloadPanelVram: "ذاكرة الفيديو",
+ offloadPanelDepth: "عمق الجلب المسبق",
+ offloadPanelPinned: "مثبّتة",
+ offloadPanelSweepNote: "تُظهر النافذة المتحركة ترتيب جلب الطبقات بعد إبطائها لتكون مرئية؛ أما الأرقام فمقاسة في آخر خطوة.",
gradCheckpoint: "نقطة تحقق التدرّج",
gradCheckpointTooltip:
"مقايضة الحوسبة بالذاكرة عبر إعادة حساب التنشيطات.",
diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts
index 5c8274ff453..d7952038ac6 100644
--- a/studio/frontend/src/i18n/locales/de.ts
+++ b/studio/frontend/src/i18n/locales/de.ts
@@ -3452,6 +3452,28 @@ export const de = {
"Anteil der gesamten Trainingsschritte zwischen Auswertungen (0-1). Auf 0 setzen, um die Auswertung zu deaktivieren. Z. B. 0.01 = alle 1 % der Schritte auswerten.",
seed: "Seed",
seedTooltip: "Zufalls-Seed für Reproduzierbarkeit.",
+ offloadLayers: "Schichten auslagern",
+ offloadLayersTooltip: "Decoder-Schichten im Arbeitsspeicher halten und jede kurz vor ihrer Ausführung auf die GPU streamen, damit ein Modell, das größer als der VRAM ist, mit LoRA trainiert. Auto lagert so wenige wie nötig aus, eine Zahl genau so viele. Benötigt Gradient Checkpointing.",
+ offloadOff: "Aus",
+ offloadAuto: "Auto",
+ offloadCount: "Anzahl",
+ offloadVramBudget: "VRAM-Budget (GiB)",
+ offloadVramBudgetTooltip: "Der meiste VRAM, den dieser Lauf nutzen darf. Auto lagert genug Schichten aus, um darunter zu bleiben, und zwei Läufe können sich eine Karte teilen. Leer nutzt die ganze Karte.",
+ offloadWholeCard: "ganze Karte",
+ prefetchDepth: "Prefetch-Tiefe",
+ prefetchDepthTooltip: "Wie viele ausgelagerte Schichten vor der laufenden kopiert werden. Auto beginnt bei 1 und behält eine tiefere Reserve nur, solange sie das Warten auf Kopien messbar verkürzt.",
+ offloadPanelTitle: "Ausgelagerte Schichten",
+ offloadPanelGpu: "Auf der GPU",
+ offloadPanelHost: "Im Arbeitsspeicher",
+ offloadPanelCopying: "Wird kopiert",
+ offloadPanelSwapped: "ausgelagert",
+ offloadPanelStall: "Warten auf Kopien",
+ offloadPanelCopy: "Kopie pro Schicht",
+ offloadPanelCompute: "Rechnen pro Schicht",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "Prefetch-Tiefe",
+ offloadPanelPinned: "gepinnt",
+ offloadPanelSweepNote: "Das wandernde Fenster zeigt, in welcher Reihenfolge Schichten geholt werden, verlangsamt, damit man es sieht; die Zahlen sind am letzten Schritt gemessen.",
gradCheckpoint: "Grad-Checkpoint",
gradCheckpointTooltip:
"Rechenaufwand gegen Speicher tauschen, indem Aktivierungen neu berechnet werden.",
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index f14ad81fd67..b9fff8e8ce9 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -3456,6 +3456,28 @@ export const en = {
"Fraction of total training steps between evaluations (0-1). Set to 0 to disable evaluation. E.g. 0.01 = evaluate every 1% of steps.",
seed: "Seed",
seedTooltip: "Random seed for reproducibility.",
+ offloadLayers: "Offload layers",
+ offloadLayersTooltip: "Keep decoder layers in system RAM and stream each one to the GPU just before it runs, so a model larger than your VRAM trains with LoRA. Auto offloads as few as fit; a count offloads exactly that many. Needs gradient checkpointing.",
+ offloadOff: "Off",
+ offloadAuto: "Auto",
+ offloadCount: "Count",
+ offloadVramBudget: "VRAM budget (GiB)",
+ offloadVramBudgetTooltip: "The most VRAM this run may use. Auto offloads enough layers to fit under it, and two runs can share one card by each taking part of it. Empty uses the whole card.",
+ offloadWholeCard: "whole card",
+ prefetchDepth: "Prefetch depth",
+ prefetchDepthTooltip: "How many offloaded layers are copied ahead of the one running. Auto starts at 1 and keeps a deeper pool only while it measurably cuts waiting on copies.",
+ offloadPanelTitle: "Offloaded layers",
+ offloadPanelGpu: "On GPU",
+ offloadPanelHost: "In system RAM",
+ offloadPanelCopying: "Copying",
+ offloadPanelSwapped: "offloaded",
+ offloadPanelStall: "waiting on copies",
+ offloadPanelCopy: "copy per layer",
+ offloadPanelCompute: "compute per layer",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "prefetch depth",
+ offloadPanelPinned: "pinned",
+ offloadPanelSweepNote: "The moving window shows the order layers are fetched in, slowed down to be visible; the numbers are measured on the last step.",
gradCheckpoint: "Grad Checkpoint",
gradCheckpointTooltip:
"Trade compute for memory by recomputing activations.",
diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts
index 5d6613c714b..ab9d40e0658 100644
--- a/studio/frontend/src/i18n/locales/es.ts
+++ b/studio/frontend/src/i18n/locales/es.ts
@@ -3447,6 +3447,28 @@ export const es = {
"Fracción del total de pasos de entrenamiento entre evaluaciones (0-1). Pon 0 para desactivar la evaluación. P. ej. 0.01 = evaluar cada 1 % de los pasos.",
seed: "Semilla",
seedTooltip: "Semilla aleatoria para reproducibilidad.",
+ offloadLayers: "Descargar capas",
+ offloadLayersTooltip: "Mantiene capas del decodificador en la RAM del sistema y transfiere cada una a la GPU justo antes de ejecutarla, para entrenar con LoRA un modelo más grande que tu VRAM. Auto descarga las mínimas que hagan falta; un número descarga exactamente esas. Requiere gradient checkpointing.",
+ offloadOff: "Desactivado",
+ offloadAuto: "Auto",
+ offloadCount: "Número",
+ offloadVramBudget: "Presupuesto de VRAM (GiB)",
+ offloadVramBudgetTooltip: "La VRAM máxima que puede usar este entrenamiento. Auto descarga las capas necesarias para caber, y dos entrenamientos pueden compartir una tarjeta. Vacío usa toda la tarjeta.",
+ offloadWholeCard: "toda la tarjeta",
+ prefetchDepth: "Profundidad de precarga",
+ prefetchDepthTooltip: "Cuántas capas descargadas se copian antes de la que se ejecuta. Auto empieza en 1 y solo mantiene más si reduce de forma medible la espera de copias.",
+ offloadPanelTitle: "Capas descargadas",
+ offloadPanelGpu: "En la GPU",
+ offloadPanelHost: "En la RAM del sistema",
+ offloadPanelCopying: "Copiando",
+ offloadPanelSwapped: "descargadas",
+ offloadPanelStall: "esperando copias",
+ offloadPanelCopy: "copia por capa",
+ offloadPanelCompute: "cálculo por capa",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "profundidad de precarga",
+ offloadPanelPinned: "fijada",
+ offloadPanelSweepNote: "La ventana móvil muestra el orden en que se traen las capas, ralentizado para que se vea; los números se miden en el último paso.",
gradCheckpoint: "Checkpoint de gradiente",
gradCheckpointTooltip:
"Intercambia cómputo por memoria recalculando las activaciones.",
diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts
index cfa0bf1d02d..be638a542ec 100644
--- a/studio/frontend/src/i18n/locales/fr.ts
+++ b/studio/frontend/src/i18n/locales/fr.ts
@@ -3462,6 +3462,28 @@ export const fr = {
"Fraction du nombre total d'étapes d'entraînement entre les évaluations (0-1). Définissez 0 pour désactiver l'évaluation. Ex. 0,01 = évaluer tous les 1 % d'étapes.",
seed: "Graine",
seedTooltip: "Graine aléatoire pour la reproductibilité.",
+ offloadLayers: "Décharger des couches",
+ offloadLayersTooltip: "Garde des couches du décodeur en RAM système et transfère chacune vers le GPU juste avant son exécution, pour entraîner avec LoRA un modèle plus grand que votre VRAM. Auto en décharge le moins possible ; un nombre en décharge exactement autant. Nécessite le gradient checkpointing.",
+ offloadOff: "Désactivé",
+ offloadAuto: "Auto",
+ offloadCount: "Nombre",
+ offloadVramBudget: "Budget VRAM (Gio)",
+ offloadVramBudgetTooltip: "La VRAM maximale que cet entraînement peut utiliser. Auto décharge assez de couches pour rester en dessous, et deux entraînements peuvent partager une carte. Vide utilise toute la carte.",
+ offloadWholeCard: "toute la carte",
+ prefetchDepth: "Profondeur de préchargement",
+ prefetchDepthTooltip: "Nombre de couches déchargées copiées avant celle qui s'exécute. Auto commence à 1 et ne garde une réserve plus profonde que si elle réduit l'attente des copies de façon mesurable.",
+ offloadPanelTitle: "Couches déchargées",
+ offloadPanelGpu: "Sur le GPU",
+ offloadPanelHost: "En RAM système",
+ offloadPanelCopying: "Copie en cours",
+ offloadPanelSwapped: "déchargées",
+ offloadPanelStall: "attente des copies",
+ offloadPanelCopy: "copie par couche",
+ offloadPanelCompute: "calcul par couche",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "profondeur de préchargement",
+ offloadPanelPinned: "épinglée",
+ offloadPanelSweepNote: "La fenêtre mobile montre l'ordre de chargement des couches, ralenti pour être visible ; les chiffres sont mesurés sur la dernière étape.",
gradCheckpoint: "Checkpoint de gradient",
gradCheckpointTooltip:
"Échanger du calcul contre de la mémoire en recalculant les activations.",
diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.ts
index 4b2cd54ca97..44c7dfd31f7 100644
--- a/studio/frontend/src/i18n/locales/he.ts
+++ b/studio/frontend/src/i18n/locales/he.ts
@@ -3492,6 +3492,28 @@ export const he = {
"חלק מסך כל צעדי האימון בין הערכות (0-1). הגדר 0 כדי להשבית הערכה. לדוגמה 0.01 = הערכה בכל 1% מהצעדים.",
seed: "גרעין אקראיות (Seed)",
seedTooltip: "גרעין אקראי לשחזור מדויק של תוצאות.",
+ offloadLayers: "העברת שכבות לזיכרון המערכת",
+ offloadLayersTooltip: "שמירת שכבות המפענח בזיכרון המערכת והזרמת כל אחת ל-GPU רגע לפני שהיא רצה, כך שמודל גדול מה-VRAM שלך מתאמן עם LoRA. אוטומטי מעביר כמה שפחות שכבות שנדרש; מספר מעביר בדיוק כמה שצוין. דורש gradient checkpointing.",
+ offloadOff: "כבוי",
+ offloadAuto: "אוטומטי",
+ offloadCount: "מספר",
+ offloadVramBudget: "תקציב VRAM ‏(GiB)",
+ offloadVramBudgetTooltip: "כמות ה-VRAM המרבית שהריצה הזו רשאית להשתמש בה. אוטומטי מעביר מספיק שכבות כדי להיכנס מתחתיה, ושתי ריצות יכולות לחלוק כרטיס אחד כשכל אחת לוקחת חלק ממנו. שדה ריק משתמש בכרטיס כולו.",
+ offloadWholeCard: "הכרטיס כולו",
+ prefetchDepth: "עומק טעינה מוקדמת",
+ prefetchDepthTooltip: "כמה שכבות מועברות מועתקות מראש לפני השכבה שרצה. אוטומטי מתחיל ב-1 ושומר מאגר עמוק יותר רק כל עוד הוא מקצר בפועל את ההמתנה להעתקות.",
+ offloadPanelTitle: "שכבות שהועברו",
+ offloadPanelGpu: "ב-GPU",
+ offloadPanelHost: "בזיכרון המערכת",
+ offloadPanelCopying: "מועתקת",
+ offloadPanelSwapped: "הועברו",
+ offloadPanelStall: "המתנה להעתקות",
+ offloadPanelCopy: "העתקה לשכבה",
+ offloadPanelCompute: "חישוב לשכבה",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "עומק טעינה מוקדמת",
+ offloadPanelPinned: "נעול בזיכרון",
+ offloadPanelSweepNote: "החלון הנע מראה את הסדר שבו השכבות נטענות, מואט כדי שיהיה אפשר לראותו; המספרים נמדדו בצעד האחרון.",
gradCheckpoint: "נקודות ביקורת לגרדיאנט",
gradCheckpointTooltip:
"פשרה בין זמן חישוב לזיכרון על ידי חישוב מחדש של אקטיבציות.",
diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts
index 69bba7c501e..eb18b863cf5 100644
--- a/studio/frontend/src/i18n/locales/hi.ts
+++ b/studio/frontend/src/i18n/locales/hi.ts
@@ -3401,6 +3401,28 @@ export const hi = {
"मूल्यांकनों के बीच कुल ट्रेनिंग स्टेप्स का अंश (0-1)। मूल्यांकन अक्षम करने के लिए 0 सेट करें। उदा. 0.01 = हर 1% स्टेप्स पर मूल्यांकन करें।",
seed: "सीड",
seedTooltip: "पुनरुत्पादन के लिए रैंडम seed।",
+ offloadLayers: "परतें ऑफ़लोड करें",
+ offloadLayersTooltip: "डिकोडर परतों को सिस्टम RAM में रखें और हर परत को चलने से ठीक पहले GPU पर भेजें, ताकि VRAM से बड़ा मॉडल LoRA के साथ ट्रेन हो सके। ऑटो जितनी कम ज़रूरी हों उतनी ऑफ़लोड करता है; संख्या ठीक उतनी। ग्रेडिएंट चेकपॉइंटिंग आवश्यक है।",
+ offloadOff: "बंद",
+ offloadAuto: "ऑटो",
+ offloadCount: "संख्या",
+ offloadVramBudget: "VRAM बजट (GiB)",
+ offloadVramBudgetTooltip: "यह रन अधिकतम कितना VRAM इस्तेमाल कर सकता है। ऑटो इसके भीतर रहने लायक परतें ऑफ़लोड करता है, और दो रन एक कार्ड बाँट सकते हैं। खाली छोड़ने पर पूरा कार्ड।",
+ offloadWholeCard: "पूरा कार्ड",
+ prefetchDepth: "प्रीफ़ेच गहराई",
+ prefetchDepthTooltip: "चल रही परत से पहले कितनी ऑफ़लोड परतें कॉपी हों। ऑटो 1 से शुरू होता है और गहरा पूल तभी रखता है जब वह कॉपी का इंतज़ार सच में घटाए।",
+ offloadPanelTitle: "ऑफ़लोड परतें",
+ offloadPanelGpu: "GPU पर",
+ offloadPanelHost: "सिस्टम RAM में",
+ offloadPanelCopying: "कॉपी हो रही",
+ offloadPanelSwapped: "ऑफ़लोड",
+ offloadPanelStall: "कॉपी का इंतज़ार",
+ offloadPanelCopy: "प्रति परत कॉपी",
+ offloadPanelCompute: "प्रति परत गणना",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "प्रीफ़ेच गहराई",
+ offloadPanelPinned: "पिन की गई",
+ offloadPanelSweepNote: "चलती खिड़की दिखाती है कि परतें किस क्रम में लाई जाती हैं, दिखने लायक धीमा करके; संख्याएँ पिछले स्टेप पर मापी गई हैं।",
gradCheckpoint: "ग्रेडिएंट चेकपॉइंट",
gradCheckpointTooltip:
"एक्टिवेशन की दोबारा गणना करके अतिरिक्त कंप्यूट के बदले मेमोरी बचाएँ।",
diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts
index f31ea39c2f7..c1294d67005 100644
--- a/studio/frontend/src/i18n/locales/it.ts
+++ b/studio/frontend/src/i18n/locales/it.ts
@@ -3455,6 +3455,28 @@ export const it = {
"Frazione degli step totali fra due valutazioni (0-1). Imposta 0 per disattivare la valutazione. Per esempio 0,01 = valuta ogni 1% degli step.",
seed: "Seed",
seedTooltip: "Seed casuale per la riproducibilità.",
+ offloadLayers: "Scarica livelli",
+ offloadLayersTooltip: "Mantiene i livelli del decoder nella RAM di sistema e trasferisce ciascuno alla GPU subito prima dell'esecuzione, così un modello più grande della tua VRAM si addestra con LoRA. Auto ne scarica il minimo necessario; un numero ne scarica esattamente quanti. Richiede il gradient checkpointing.",
+ offloadOff: "Disattivato",
+ offloadAuto: "Auto",
+ offloadCount: "Numero",
+ offloadVramBudget: "Budget VRAM (GiB)",
+ offloadVramBudgetTooltip: "La VRAM massima che questo addestramento può usare. Auto scarica abbastanza livelli per restare sotto, e due addestramenti possono condividere una scheda. Vuoto usa tutta la scheda.",
+ offloadWholeCard: "tutta la scheda",
+ prefetchDepth: "Profondità di prefetch",
+ prefetchDepthTooltip: "Quanti livelli scaricati vengono copiati prima di quello in esecuzione. Auto parte da 1 e tiene una riserva più profonda solo se riduce in modo misurabile l'attesa delle copie.",
+ offloadPanelTitle: "Livelli scaricati",
+ offloadPanelGpu: "Sulla GPU",
+ offloadPanelHost: "Nella RAM di sistema",
+ offloadPanelCopying: "In copia",
+ offloadPanelSwapped: "scaricati",
+ offloadPanelStall: "attesa delle copie",
+ offloadPanelCopy: "copia per livello",
+ offloadPanelCompute: "calcolo per livello",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "profondità di prefetch",
+ offloadPanelPinned: "bloccata",
+ offloadPanelSweepNote: "La finestra mobile mostra l'ordine in cui i livelli vengono caricati, rallentato per essere visibile; i numeri sono misurati sull'ultimo passo.",
gradCheckpoint: "Checkpoint del gradiente",
gradCheckpointTooltip:
"Riduce l'uso della memoria ricalcolando le attivazioni, al costo di più calcoli.",
diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts
index cdca1d3a0d4..204bdb50161 100644
--- a/studio/frontend/src/i18n/locales/ja.ts
+++ b/studio/frontend/src/i18n/locales/ja.ts
@@ -3346,6 +3346,28 @@ export const ja = {
evalStepsTooltip: "評価を行う間隔を、総トレーニングステップに対する割合 (0-1) で指定します。0 に設定すると評価が無効になります。例: 0.01 = ステップの 1% ごとに評価を実行。",
seed: "シード値",
seedTooltip: "再現性のための乱数シード。",
+ offloadLayers: "レイヤーのオフロード",
+ offloadLayersTooltip: "デコーダー層をシステムRAMに置き、各層を実行直前にGPUへ転送します。VRAMより大きなモデルをLoRAで学習できます。自動は必要最小限だけ、数値はその数だけオフロードします。勾配チェックポイントが必要です。",
+ offloadOff: "オフ",
+ offloadAuto: "自動",
+ offloadCount: "数",
+ offloadVramBudget: "VRAM予算 (GiB)",
+ offloadVramBudgetTooltip: "この学習が使えるVRAMの上限です。自動はその範囲に収まるようにオフロードし、2つの学習で1枚のカードを分け合えます。空欄ならカード全体を使います。",
+ offloadWholeCard: "カード全体",
+ prefetchDepth: "プリフェッチ深度",
+ prefetchDepthTooltip: "実行中の層より先にコピーするオフロード層の数です。自動は1から始め、コピー待ちが実際に減る場合だけ深くします。",
+ offloadPanelTitle: "オフロード層",
+ offloadPanelGpu: "GPU上",
+ offloadPanelHost: "システムRAM",
+ offloadPanelCopying: "コピー中",
+ offloadPanelSwapped: "オフロード",
+ offloadPanelStall: "コピー待ち",
+ offloadPanelCopy: "層ごとのコピー",
+ offloadPanelCompute: "層ごとの計算",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "プリフェッチ深度",
+ offloadPanelPinned: "ピン留め",
+ offloadPanelSweepNote: "移動するウィンドウは層を取り込む順序を、見えるように遅くして示します。数値は直前のステップの実測値です。",
gradCheckpoint: "勾配チェックポイント",
gradCheckpointTooltip: "アクティベーションを再計算することで、計算量と引き換えにメモリを節約します。",
none: "なし",
diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts
index 63d4703564c..425dd25ca02 100644
--- a/studio/frontend/src/i18n/locales/ko.ts
+++ b/studio/frontend/src/i18n/locales/ko.ts
@@ -3385,6 +3385,28 @@ export const ko = {
"평가 사이의 간격을 전체 학습 스텝 수의 비율로 나타냅니다(0-1). 0으로 설정하면 평가가 비활성화됩니다. 예: 0.01 = 스텝의 1%마다 평가.",
seed: "시드",
seedTooltip: "재현성을 위한 랜덤 시드입니다.",
+ offloadLayers: "레이어 오프로드",
+ offloadLayersTooltip: "디코더 레이어를 시스템 RAM에 두고 각 레이어를 실행 직전에 GPU로 전송해, VRAM보다 큰 모델을 LoRA로 학습합니다. 자동은 필요한 만큼만, 숫자는 정확히 그만큼 오프로드합니다. 그래디언트 체크포인팅이 필요합니다.",
+ offloadOff: "끄기",
+ offloadAuto: "자동",
+ offloadCount: "개수",
+ offloadVramBudget: "VRAM 예산 (GiB)",
+ offloadVramBudgetTooltip: "이 학습이 쓸 수 있는 최대 VRAM입니다. 자동은 그 안에 맞도록 레이어를 오프로드하고, 두 학습이 카드 하나를 나눠 쓸 수 있습니다. 비우면 카드 전체를 씁니다.",
+ offloadWholeCard: "카드 전체",
+ prefetchDepth: "프리페치 깊이",
+ prefetchDepthTooltip: "실행 중인 레이어보다 먼저 복사할 오프로드 레이어 수입니다. 자동은 1에서 시작해 복사 대기가 실제로 줄 때만 깊게 유지합니다.",
+ offloadPanelTitle: "오프로드된 레이어",
+ offloadPanelGpu: "GPU에 있음",
+ offloadPanelHost: "시스템 RAM에 있음",
+ offloadPanelCopying: "복사 중",
+ offloadPanelSwapped: "오프로드됨",
+ offloadPanelStall: "복사 대기",
+ offloadPanelCopy: "레이어당 복사",
+ offloadPanelCompute: "레이어당 연산",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "프리페치 깊이",
+ offloadPanelPinned: "고정됨",
+ offloadPanelSweepNote: "움직이는 창은 레이어를 가져오는 순서를 보이도록 느리게 보여 줍니다. 숫자는 마지막 스텝에서 측정한 값입니다.",
gradCheckpoint: "그래디언트 체크포인팅",
gradCheckpointTooltip:
"활성화를 재계산해 연산량을 늘리는 대신 메모리 사용량을 줄입니다.",
diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts
index 077cea05f07..3e6bbd51b27 100644
--- a/studio/frontend/src/i18n/locales/pt-br.ts
+++ b/studio/frontend/src/i18n/locales/pt-br.ts
@@ -3418,6 +3418,28 @@ export const ptBR = {
"Fração dos passos totais de treino entre as validações (0-1). Defina como 0 para desativar. Ex: 0.01 = valida a cada 1% dos passos.",
seed: "Seed",
seedTooltip: "Semente aleatória para reprodutibilidade.",
+ offloadLayers: "Descarregar camadas",
+ offloadLayersTooltip: "Mantém camadas do decodificador na RAM do sistema e envia cada uma para a GPU logo antes de executá-la, para treinar com LoRA um modelo maior que sua VRAM. Auto descarrega o mínimo necessário; um número descarrega exatamente esse total. Requer gradient checkpointing.",
+ offloadOff: "Desligado",
+ offloadAuto: "Auto",
+ offloadCount: "Número",
+ offloadVramBudget: "Orçamento de VRAM (GiB)",
+ offloadVramBudgetTooltip: "A VRAM máxima que este treino pode usar. Auto descarrega camadas suficientes para caber, e dois treinos podem dividir uma placa. Vazio usa a placa inteira.",
+ offloadWholeCard: "placa inteira",
+ prefetchDepth: "Profundidade de pré-busca",
+ prefetchDepthTooltip: "Quantas camadas descarregadas são copiadas antes da que está rodando. Auto começa em 1 e só mantém mais se reduzir de forma mensurável a espera por cópias.",
+ offloadPanelTitle: "Camadas descarregadas",
+ offloadPanelGpu: "Na GPU",
+ offloadPanelHost: "Na RAM do sistema",
+ offloadPanelCopying: "Copiando",
+ offloadPanelSwapped: "descarregadas",
+ offloadPanelStall: "esperando cópias",
+ offloadPanelCopy: "cópia por camada",
+ offloadPanelCompute: "cálculo por camada",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "profundidade de pré-busca",
+ offloadPanelPinned: "fixada",
+ offloadPanelSweepNote: "A janela móvel mostra a ordem em que as camadas são buscadas, desacelerada para ficar visível; os números são medidos no último passo.",
gradCheckpoint: "Checkpoint de gradiente",
gradCheckpointTooltip:
"Reduz o uso de memória ao recalcular as ativações, em troca de maior custo computacional.",
diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts
index e570de4b2f3..04f335c716d 100644
--- a/studio/frontend/src/i18n/locales/ru.ts
+++ b/studio/frontend/src/i18n/locales/ru.ts
@@ -3417,6 +3417,28 @@ export const ru = {
"Доля от общего числа шагов обучения между оценками (0-1). Задайте 0, чтобы отключить оценку. Например, 0.01 = оценка каждый 1% шагов.",
seed: "Seed",
seedTooltip: "Случайный seed для воспроизводимости.",
+ offloadLayers: "Выгрузка слоёв",
+ offloadLayersTooltip: "Хранить слои декодера в оперативной памяти и передавать каждый на GPU непосредственно перед выполнением, чтобы модель больше видеопамяти обучалась с LoRA. Авто выгружает минимум необходимого, число выгружает ровно столько. Требуется gradient checkpointing.",
+ offloadOff: "Выкл",
+ offloadAuto: "Авто",
+ offloadCount: "Число",
+ offloadVramBudget: "Бюджет видеопамяти (ГиБ)",
+ offloadVramBudgetTooltip: "Максимум видеопамяти для этого обучения. Авто выгружает столько слоёв, чтобы уложиться, и два обучения могут делить одну карту. Пусто: вся карта.",
+ offloadWholeCard: "вся карта",
+ prefetchDepth: "Глубина предзагрузки",
+ prefetchDepthTooltip: "Сколько выгруженных слоёв копируется заранее. Авто начинает с 1 и держит больший запас, только если он заметно сокращает ожидание копий.",
+ offloadPanelTitle: "Выгруженные слои",
+ offloadPanelGpu: "На GPU",
+ offloadPanelHost: "В оперативной памяти",
+ offloadPanelCopying: "Копируется",
+ offloadPanelSwapped: "выгружено",
+ offloadPanelStall: "ожидание копий",
+ offloadPanelCopy: "копия на слой",
+ offloadPanelCompute: "вычисление на слой",
+ offloadPanelVram: "Видеопамять",
+ offloadPanelDepth: "глубина предзагрузки",
+ offloadPanelPinned: "закреплено",
+ offloadPanelSweepNote: "Движущееся окно показывает порядок загрузки слоёв, замедленно, чтобы его было видно; числа измерены на последнем шаге.",
gradCheckpoint: "Чекпоинт градиента",
gradCheckpointTooltip:
"Обменять вычисления на память, пересчитывая активации.",
diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.ts
index 8555383dd47..36bf414e455 100644
--- a/studio/frontend/src/i18n/locales/sv.ts
+++ b/studio/frontend/src/i18n/locales/sv.ts
@@ -3613,6 +3613,28 @@ export const sv = {
"Andel av totala träningssteg mellan utvärderingar (0–1). Ange 0 för att inaktivera utvärdering. Exempel: 0,01 = utvärdera var 1 % av stegen.",
seed: "Slumptalets startvärde",
seedTooltip: "Slumptalets startvärde för reproducerbarhet.",
+ offloadLayers: "Avlasta lager",
+ offloadLayersTooltip: "Håll avkodarlager i systemminnet och strömma vart och ett till GPU:n precis innan det körs, så att en modell som är större än ditt VRAM kan tränas med LoRA. Auto avlastar så få som behövs; ett antal avlastar exakt så många. Kräver gradient checkpointing.",
+ offloadOff: "Av",
+ offloadAuto: "Auto",
+ offloadCount: "Antal",
+ offloadVramBudget: "VRAM-budget (GiB)",
+ offloadVramBudgetTooltip: "Det mesta VRAM den här körningen får använda. Auto avlastar tillräckligt många lager för att hålla sig under det, och två körningar kan dela ett kort genom att ta varsin del. Tomt använder hela kortet.",
+ offloadWholeCard: "hela kortet",
+ prefetchDepth: "Förhämtningsdjup",
+ prefetchDepthTooltip: "Hur många avlastade lager som kopieras i förväg före det som körs. Auto börjar på 1 och behåller en djupare pool bara så länge den mätbart minskar väntan på kopior.",
+ offloadPanelTitle: "Avlastade lager",
+ offloadPanelGpu: "På GPU",
+ offloadPanelHost: "I systemminnet",
+ offloadPanelCopying: "Kopieras",
+ offloadPanelSwapped: "avlastade",
+ offloadPanelStall: "väntan på kopior",
+ offloadPanelCopy: "kopiering per lager",
+ offloadPanelCompute: "beräkning per lager",
+ offloadPanelVram: "VRAM",
+ offloadPanelDepth: "förhämtningsdjup",
+ offloadPanelPinned: "låst",
+ offloadPanelSweepNote: "Det rörliga fönstret visar i vilken ordning lagren hämtas, saktat ner för att synas; siffrorna är uppmätta på det senaste steget.",
gradCheckpoint: "Gradientkontrollpunkt",
gradCheckpointTooltip:
"Byt beräkningskraft mot minne genom att beräkna aktiveringar igen.",
diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts
index 1407cf90042..ccd30b60aaa 100644
--- a/studio/frontend/src/i18n/locales/zh-CN.ts
+++ b/studio/frontend/src/i18n/locales/zh-CN.ts
@@ -3311,6 +3311,28 @@ export const zhCN = {
"评估之间间隔占总训练步数的比例(0-1)。设为 0 则禁用评估。例如 0.01 = 每 1% 步评估一次。",
seed: "随机种子",
seedTooltip: "用于复现的随机种子。",
+ offloadLayers: "卸载层",
+ offloadLayersTooltip: "将解码器层保存在系统内存中,并在每层运行前传输到 GPU,从而用 LoRA 训练比显存更大的模型。自动只卸载必要的最少层数;填写数字则卸载该数量。需要启用梯度检查点。",
+ offloadOff: "关闭",
+ offloadAuto: "自动",
+ offloadCount: "数量",
+ offloadVramBudget: "显存预算 (GiB)",
+ offloadVramBudgetTooltip: "本次训练最多可使用的显存。自动会卸载足够的层以保持在此之下,两次训练也可以分享同一张显卡。留空则使用整张显卡。",
+ offloadWholeCard: "整张显卡",
+ prefetchDepth: "预取深度",
+ prefetchDepthTooltip: "在当前层之前提前复制的卸载层数量。自动从 1 开始,只有在确实减少复制等待时才加深。",
+ offloadPanelTitle: "卸载的层",
+ offloadPanelGpu: "在 GPU 上",
+ offloadPanelHost: "在系统内存中",
+ offloadPanelCopying: "复制中",
+ offloadPanelSwapped: "已卸载",
+ offloadPanelStall: "等待复制",
+ offloadPanelCopy: "每层复制",
+ offloadPanelCompute: "每层计算",
+ offloadPanelVram: "显存",
+ offloadPanelDepth: "预取深度",
+ offloadPanelPinned: "已固定",
+ offloadPanelSweepNote: "移动窗口放慢显示各层的取入顺序;数字是上一步的实测值。",
gradCheckpoint: "梯度检查点",
gradCheckpointTooltip: "通过重算激活以时间换 VRAM。",
none: "无",
diff --git a/studio/frontend/tests/offload-panel-placement.test.ts b/studio/frontend/tests/offload-panel-placement.test.ts
new file mode 100644
index 00000000000..5d8cd956d79
--- /dev/null
+++ b/studio/frontend/tests/offload-panel-placement.test.ts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// The live offload panel's grid: layers never offloaded stay resident, the running layer and the
+// next `depth` offloaded ones are on the card, every other offloaded layer is in system RAM.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { layerPlacement, offloadCountFromInput } = await import("../src/features/studio/sections/offload-panel-layout.ts");
+
+test("the window follows the offloaded order, not the layer index", () => {
+ // Spread placement: every other layer offloaded.
+ const cells = layerPlacement(8, [1, 3, 5, 7], 2, 1);
+ assert.deepEqual(cells, [
+ "resident", "host", "resident", "running", "resident", "copying", "resident", "copying",
+ ]);
+});
+
+test("the window stops at the last offloaded layer", () => {
+ const cells = layerPlacement(6, [3, 4, 5], 2, 2);
+ assert.deepEqual(cells, ["resident", "resident", "resident", "host", "host", "running"]);
+});
+
+test("nothing offloaded is all resident", () => {
+ assert.deepEqual(layerPlacement(3, [], 2, 0), ["resident", "resident", "resident"]);
+});
+
+test("clearing the Count box keeps the last count instead of turning offload off", () => {
+ for (const text of ["", "0", "-3", "abc"]) assert.equal(offloadCountFromInput(text), null);
+ assert.equal(offloadCountFromInput("14"), 14);
+ assert.equal(offloadCountFromInput("6.7"), 6);
+});
diff --git a/studio/frontend/tests/training-model-defaults-offload.test.ts b/studio/frontend/tests/training-model-defaults-offload.test.ts
new file mode 100644
index 00000000000..ebb016a02c1
--- /dev/null
+++ b/studio/frontend/tests/training-model-defaults-offload.test.ts
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// The YAML export writes every offload field, so an import has to read every one back.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerStoreStubResolver } from "./helpers/kit.ts";
+
+registerStoreStubResolver();
+
+const { mapBackendModelConfigToTrainingPatch } = await import(
+ "../src/features/training/lib/model-defaults.ts"
+);
+
+test("importing an exported config restores every offload field", () => {
+ const patch = mapBackendModelConfigToTrainingPatch({
+ training: { offload_layers: "auto", offload_vram_gb: 12.5, prefetch_depth: "auto" },
+ });
+ assert.equal(patch.offloadLayers, "auto");
+ assert.equal(patch.offloadVramGb, 12.5);
+ assert.equal(patch.prefetchDepth, "auto");
+ const off = mapBackendModelConfigToTrainingPatch({
+ training: { offload_layers: 14, offload_vram_gb: null, prefetch_depth: 3 },
+ });
+ assert.equal(off.offloadVramGb, null);
+ assert.equal(off.prefetchDepth, 3);
+ // Configs exported before these fields existed keep the current values.
+ assert.equal("offloadVramGb" in mapBackendModelConfigToTrainingPatch({ training: {} }), false);
+});
diff --git a/studio/frontend/tests/training-start-payload-offload.test.ts b/studio/frontend/tests/training-start-payload-offload.test.ts
new file mode 100644
index 00000000000..3199667f422
--- /dev/null
+++ b/studio/frontend/tests/training-start-payload-offload.test.ts
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Offload layers in the start payload: off by default, a full finetune always sends it off,
+// and the VRAM budget goes out only with "auto", the one mode that sizes to it.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { offloadHardwareSupported, offloadPayload, offloadSupported } = await import("../src/features/training/api/mappers.ts");
+const { initialTrainingConfigState } = await import(
+ "../src/features/training/stores/training-config-policy.ts"
+);
+
+const base = { ...initialTrainingConfigState, trainingMethod: "lora" as const };
+
+test("defaults send offload off at depth 2", () => {
+ assert.deepEqual(offloadPayload(base), {
+ offload_layers: 0,
+ offload_vram_gb: null,
+ prefetch_depth: 2,
+ });
+});
+
+test("auto carries the budget and an auto depth", () => {
+ assert.deepEqual(
+ offloadPayload({ ...base, offloadLayers: "auto", offloadVramGb: 11.5, prefetchDepth: "auto" }),
+ { offload_layers: "auto", offload_vram_gb: 11.5, prefetch_depth: "auto" },
+ );
+});
+
+test("a fixed count drops the budget and rounds the count", () => {
+ assert.deepEqual(offloadPayload({ ...base, offloadLayers: 14.7, offloadVramGb: 8, prefetchDepth: 3 }), {
+ offload_layers: 14,
+ offload_vram_gb: null,
+ prefetch_depth: 3,
+ });
+});
+
+test("a full finetune never offloads", () => {
+ const out = offloadPayload({ ...base, trainingMethod: "full", offloadLayers: "auto", offloadVramGb: 8 });
+ assert.equal(out.offload_layers, 0);
+ assert.equal(out.offload_vram_gb, null);
+});
+
+test("depth is clamped to what the backend accepts", () => {
+ assert.equal(offloadPayload({ ...base, offloadLayers: 4, prefetchDepth: 40 }).prefetch_depth, 8);
+ assert.equal(offloadPayload({ ...base, offloadLayers: 4, prefetchDepth: 0 }).prefetch_depth, 2);
+});
+
+test("offload goes out off wherever the run cannot offload", () => {
+ const on = { ...base, offloadLayers: "auto" as const, offloadVramGb: 8 };
+ for (const blocked of [
+ { gradientCheckpointing: "none" as const },
+ { isEmbeddingModel: true },
+ { isAudioModel: true },
+ { modelType: "decision" as const },
+ { modelType: "embeddings" as const },
+ ]) {
+ assert.equal(offloadSupported({ ...on, ...blocked }), false);
+ assert.deepEqual(offloadPayload({ ...on, ...blocked }).offload_layers, 0);
+ assert.equal(offloadPayload({ ...on, ...blocked }).offload_vram_gb, null);
+ }
+ assert.equal(offloadSupported(on), true);
+});
+
+test("a count above the backend limit is clamped instead of failing the start", () => {
+ assert.equal(offloadPayload({ ...base, offloadLayers: 5000 }).offload_layers, 1024);
+});
+
+test("offload needs a CUDA or ROCm card that is not a unified-memory APU", () => {
+ const sys = (device_backend: string, unified: boolean[]) => ({
+ status: "ready" as const,
+ device_backend: device_backend as "cuda",
+ gpu: { available: true, devices: unified.map((u, index) => ({ index, unified_memory: u })) },
+ });
+ assert.equal(offloadHardwareSupported(sys("cuda", [false])), true);
+ assert.equal(offloadHardwareSupported(sys("rocm", [false])), true);
+ for (const backend of ["xpu", "mlx", "cpu"]) {
+ assert.equal(offloadHardwareSupported(sys(backend, [false])), false);
+ }
+ // A ROCm APU alone (Strix Halo) has no separate pool; beside a discrete card it does.
+ assert.equal(offloadHardwareSupported(sys("rocm", [true])), false);
+ assert.equal(offloadHardwareSupported(sys("rocm", [true, false])), true);
+ assert.equal(offloadHardwareSupported(null), true);
+ assert.equal(offloadHardwareSupported({ ...sys("cpu", []), status: "pending" }), true);
+ // A hidden setting is sent off too, so a saved Count cannot fail the run.
+ const saved = { ...base, offloadLayers: 14 };
+ assert.equal(offloadPayload(saved, sys("xpu", [false])).offload_layers, 0);
+ assert.equal(offloadPayload(saved, sys("rocm", [true])).offload_layers, 0);
+ assert.equal(offloadPayload(saved, sys("cuda", [false])).offload_layers, 14);
+});
diff --git a/tests/studio/test_hf_token_validation_tick_contract.py b/tests/studio/test_hf_token_validation_tick_contract.py
index d5cd31a9835..5169a00d892 100644
--- a/tests/studio/test_hf_token_validation_tick_contract.py
+++ b/tests/studio/test_hf_token_validation_tick_contract.py
@@ -161,7 +161,10 @@ def test_training_start_prepares_token_once_before_transport():
"async function checkSelectedDataset", 1
)[0]
assert "await prepareHfTokenForUse(attempt.hfToken)" in prepare_attempt
- assert "buildTrainingStartPayload(attempt.config, hfToken)" in submit_attempt
+ assert (
+ "buildTrainingStartPayload(attempt.config, hfToken, getCachedSystemInfo())"
+ in submit_attempt
+ )
assert "payload.hf_token = hfToken" not in submit_attempt
assert submit_attempt.index("attempt.enterTransport()") < submit_attempt.index(
"await startTraining(payload, attempt.startRequestId)"
diff --git a/tests/test_block_swap_install.py b/tests/test_block_swap_install.py
index 4bd57fa3476..f9babd4082e 100644
--- a/tests/test_block_swap_install.py
+++ b/tests/test_block_swap_install.py
@@ -15,6 +15,7 @@ NAMES = (
"_training_reserve_bytes",
"_auto_block_swap_indices",
"install_block_swap",
+ "auto_plan_depth",
"_attach_block_swap",
"trim_config_for_block_swap",
)
@@ -711,3 +712,83 @@ def test_block_swap_layers_still_works_as_the_old_name_of_offload_layers():
for path, count in (("llama.py", 2), ("vision.py", 2)):
src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
assert src.count("legacy_offload_layers(kwargs, ") == count, path
+
+
+def _fns(*names):
+ mod = ast.parse(open(UTILS, encoding = "utf-8").read())
+ ns = {}
+ body = [n for n in mod.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ exec(compile(ast.Module(body = body, type_ignores = []), UTILS, "exec"), ns)
+ return ns
+
+
[email protected]("bad", [-1, 1.5, "8", "max", True, [1, 2]])
+def test_offload_layers_refuses_values_it_cannot_use(bad):
+ with pytest.raises(ValueError, match = "offload_layers must be"):
+ _fns("legacy_offload_layers")["legacy_offload_layers"]({}, bad)
+
+
+def test_offload_layers_false_still_means_off():
+ legacy = _fns("legacy_offload_layers")["legacy_offload_layers"]
+ assert legacy({}, False) == 0
+ assert legacy({"block_swap_layers": False}) == 0
+
+
+def test_prefetch_depth_is_a_get_peft_model_option():
+ arg = _fns("prefetch_depth_arg")["prefetch_depth_arg"]
+ kwargs = {"prefetch_depth": "auto", "r": 16}
+ assert arg(kwargs) == "auto" and kwargs == {"r": 16}
+ assert arg({}) == 2 and arg({"prefetch_depth": 3}) == 3
+ for bad in (0, -1, 1.5, "fast", True):
+ with pytest.raises(ValueError, match = "prefetch_depth must be"):
+ arg({"prefetch_depth": bad})
+ # Read before the new-model route returns, and handed to every install.
+ for path, installs in (("llama.py", 2), ("vision.py", 1)):
+ src = open(os.path.join(HERE, "unsloth", "models", path), encoding = "utf-8").read()
+ assert src.count("prefetch_depth = prefetch_depth_arg(kwargs)") == 1, path
+ assert src.count("prefetch_depth = prefetch_depth,\n") >= installs, path
+
+
+def test_auto_prefetch_depth_plans_one_slot_ahead_and_reaches_the_swapper():
+ assert (
+ _fns("planned_prefetch_depth")["planned_prefetch_depth"]({"prefetch_depth": "auto"})
+ == "auto"
+ )
+ ns, calls = _load(auto_pick = ([5, 7], 0))
+ ns["BlockSwap"].stats = lambda self: {} # a zoo with adaptive depth
+ planned = []
+ ns["auto_swap_indices"] = lambda layers, reserve, depth: planned.append(depth) or ([5, 7], 0)
+ model = types.SimpleNamespace(
+ layers = _Layers(list(range(8))),
+ config = types.SimpleNamespace(),
+ parameters = lambda: [],
+ max_seq_length = 128,
+ )
+ ns["install_block_swap"](model, "auto", prefetch_depth = "auto")
+ assert planned == [1] and calls[-1][2] == "auto"
+
+
+def test_old_zoo_without_auto_depth_gets_the_default_depth():
+ ns, calls = _load()
+ ns["_new_block_swap"](_Layers([0, 1, 2]), 2, "auto", placement = "spread")
+ assert calls[-1][2] == 2
+
+
+def test_old_zoo_plans_auto_depth_at_the_two_slots_it_builds():
+ # A zoo without stats() builds "auto" as depth 2, so the plan must reserve 2, not 1.
+ ns, calls = _load(auto_pick = ([5, 7], 0))
+ planned = []
+ ns["auto_swap_indices"] = lambda layers, reserve, depth: planned.append(depth) or ([5, 7], 0)
+ model = types.SimpleNamespace(
+ layers = _Layers(list(range(8))),
+ config = types.SimpleNamespace(),
+ parameters = lambda: [],
+ max_seq_length = 128,
+ )
+ ns["install_block_swap"](model, "auto", prefetch_depth = "auto")
+ assert planned == [2] and calls[-1][2] == 2
+
+
+def test_load_time_auto_plan_uses_the_same_depth_rule():
+ src = open(os.path.join(HERE, "unsloth", "models", "loader_utils.py"), encoding = "utf-8").read()
+ assert 'options["prefetch_depth"] = auto_plan_depth(options["prefetch_depth"])' in src
diff --git a/tests/test_offload_auto_trainer_replan.py b/tests/test_offload_auto_trainer_replan.py
index 731260f05b5..399dc595fda 100644
--- a/tests/test_offload_auto_trainer_replan.py
+++ b/tests/test_offload_auto_trainer_replan.py
@@ -12,6 +12,7 @@ NAMES = (
"_training_reserve_bytes",
"_auto_block_swap_indices",
"install_block_swap",
+ "auto_plan_depth",
"_attach_block_swap",
"_layer_devices",
"_replan_auto_offload_safely",
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c9288cd1a49..b17ef82ce4d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -99,6 +99,8 @@ __all__ = [
"skip_checkpointing",
"refuse_block_swap_load",
"legacy_offload_layers",
+ "prefetch_depth_arg",
+ "auto_plan_depth",
"block_swap_load_device",
"begin_block_swap_load",
"finish_block_swap_load",
@@ -6589,6 +6591,9 @@ def _check_block_swap(model_or_config):
def _new_block_swap(layers, n, *args, placement, **kwargs):
+ if args and args[0] == "auto" and not hasattr(BlockSwap, "stats"):
+ # unsloth_zoo before prefetch_depth = "auto" takes a count only.
+ args = (2,) + args[1:]
try:
return BlockSwap(layers, n, *args, placement = placement, **kwargs)
except TypeError as e:
@@ -6602,10 +6607,29 @@ def legacy_offload_layers(kwargs, offload_layers = None):
"""`offload_layers`, or its original name `block_swap_layers` from `kwargs` when it was not given (0 = off)."""
legacy = kwargs.pop("block_swap_layers", None)
if offload_layers is None:
- return 0 if legacy is None else legacy
+ offload_layers = 0 if legacy is None else legacy
+ if offload_layers is False:
+ # False always meant off; only True is ambiguous.
+ offload_layers = 0
+ if offload_layers != "auto" and (
+ isinstance(offload_layers, bool)
+ or not isinstance(offload_layers, int)
+ or offload_layers < 0
+ ):
+ raise ValueError(
+ f"Unsloth: offload_layers must be a layer count (0 = off) or 'auto', not {offload_layers!r}."
+ )
return offload_layers
+def prefetch_depth_arg(kwargs):
+ """get_peft_model(prefetch_depth = k | "auto"): layers fetched ahead of the one running (default 2)."""
+ depth = kwargs.pop("prefetch_depth", 2)
+ if depth != "auto" and (isinstance(depth, bool) or not isinstance(depth, int) or depth < 1):
+ raise ValueError(f"Unsloth: prefetch_depth must be 1 or more, or 'auto', not {depth!r}.")
+ return depth
+
+
def refuse_block_swap_load(offload_layers, reason):
"""A load-to-host restriction: `"auto"` falls back to loading onto the GPU (0), a count raises."""
if offload_layers == "auto":
@@ -6651,9 +6675,18 @@ def begin_block_swap_load(
return load_layers_to_host(offload_layers, placement = "spread")
+def auto_plan_depth(prefetch_depth):
+ """The slot pool a swap built at `prefetch_depth` starts with, which an auto plan must reserve:
+ "auto" begins one slot ahead on a zoo that adapts, at the fixed 2 on one that does not."""
+ if prefetch_depth != "auto":
+ return prefetch_depth
+ return 1 if hasattr(BlockSwap, "stats") else 2
+
+
def planned_prefetch_depth(device_map_planner_kwargs):
# The depth "auto" sized the slot pool with; the swapper must allocate the same pool.
- return int((device_map_planner_kwargs or {}).get("prefetch_depth", 2))
+ depth = (device_map_planner_kwargs or {}).get("prefetch_depth", 2)
+ return depth if depth == "auto" else int(depth)
def finish_block_swap_load(
@@ -6887,8 +6920,9 @@ def install_block_swap(
return None
if BlockSwap is None:
_check_block_swap(model)
+ # An adaptive pool starts at one slot ahead and only grows into room it finds free.
model._unsloth_offload_layers_auto = prefetch_depth
- offload_layers = _auto_block_swap_indices(model, prefetch_depth)
+ offload_layers = _auto_block_swap_indices(model, auto_plan_depth(prefetch_depth))
if not offload_layers:
return None
_check_block_swap(model)
@@ -6998,14 +7032,19 @@ def replan_auto_offload_for_trainer(trainer):
f"get_peft_model(offload_layers = {want}) or lower per_device_train_batch_size."
)
return swapper
- elif not auto_swap_indices(layers, reserve, prefetch_depth)[0]:
+ elif not auto_swap_indices(layers, reserve, auto_plan_depth(prefetch_depth))[0]:
return swapper
if swapper is not None:
swapper.remove()
layers._unsloth_block_swap = None
model._unsloth_block_swap = None
try:
- indices = _auto_block_swap_indices(model, prefetch_depth, batch_size = rows, seq_len = seq_len)
+ indices = _auto_block_swap_indices(
+ model,
+ auto_plan_depth(prefetch_depth),
+ batch_size = rows,
+ seq_len = seq_len,
+ )
# Union: old layers stay swapped (checkpoint_skip_layers chose among the rest), new picks kept.
indices = sorted(set(old) | set(indices))
if not indices:
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 14597e90106..87e9595b67b 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -3663,6 +3663,7 @@ class FastLlamaModel:
**kwargs,
):
offload_layers = legacy_offload_layers(kwargs, offload_layers)
+ prefetch_depth = prefetch_depth_arg(kwargs)
# The flag reflects the LAST load, not this model.
_text_seq2seq = _is_text_seq2seq_config(getattr(model, "config", None))
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1" or _text_seq2seq:
@@ -3700,6 +3701,7 @@ class FastLlamaModel:
ensure_weight_tying = ensure_weight_tying,
offload_layers = offload_layers,
checkpoint_skip_layers = checkpoint_skip_layers,
+ prefetch_depth = prefetch_depth,
**kwargs,
)
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
@@ -3827,7 +3829,10 @@ class FastLlamaModel:
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
model = _exclude_rope_inv_freq_from_ddp(model)
install_block_swap(
- model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model,
+ offload_layers,
+ prefetch_depth = prefetch_depth,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
skip_checkpointing(model, checkpoint_skip_layers)
return model
@@ -4135,7 +4140,10 @@ class FastLlamaModel:
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
offload_embedding_if_tight(model)
install_block_swap(
- model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model,
+ offload_layers,
+ prefetch_depth = prefetch_depth,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
skip_checkpointing(model, checkpoint_skip_layers)
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index bfa8ae02056..d9ba756b4ef 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -768,6 +768,9 @@ def resolve_auto_block_swap(
max_memory[d] = free if cap is None else min(free, cap)
options = {k: planner_kwargs[k] for k in _BLOCK_SWAP_PLANNER_KEYS if k in planner_kwargs}
+ if "prefetch_depth" in options:
+ from ._utils import auto_plan_depth
+ options["prefetch_depth"] = auto_plan_depth(options["prefetch_depth"])
try:
planner_parameters = inspect.signature(plan_block_swap).parameters
except (TypeError, ValueError):
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index ff783f4bf4c..01f70591e96 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4173,6 +4173,7 @@ class FastBaseModel:
**kwargs,
):
offload_layers = legacy_offload_layers(kwargs, offload_layers)
+ prefetch_depth = prefetch_depth_arg(kwargs)
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
# Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the detector here too (idempotent).
@@ -4554,7 +4555,10 @@ class FastBaseModel:
module.max_seq_length = max_seq_length
offload_embedding_if_tight(model)
install_block_swap(
- model, offload_layers, use_gradient_checkpointing = use_gradient_checkpointing
+ model,
+ offload_layers,
+ prefetch_depth = prefetch_depth,
+ use_gradient_checkpointing = use_gradient_checkpointing,
)
skip_checkpointing(model, checkpoint_skip_layers)
for _ in range(3):