Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Size training VRAM with the model's attention kernel and 4-bit config (#10815)

Nilay6417d4b93577
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py
index 3b0d1aaac79..82a1c9a9aad 100644
--- a/studio/backend/tests/test_gpu_selection.py
+++ b/studio/backend/tests/test_gpu_selection.py
@@ -1751,6 +1751,31 @@ class TestMinGpuVram(unittest.TestCase):
self.assertEqual(breakdown.total, breakdown.min_gpu_vram(1))
+class TestAttentionEstimateModelClass(unittest.TestCase):
+ def _resolved_model_class(self, raw_config):
+ from utils.hardware import hardware as hardware_module
+
+ captured = {}
+
+ def _stub_resolver(model_class, cfg):
+ captured["model_class"] = model_class
+ return "sdpa"
+
+ with patch("utils.transformers_version._load_config_json", return_value = raw_config):
+ config = hardware_module._load_config_for_gpu_estimate("unsloth/test")
+ with patch.dict(sys.modules, _fake_unsloth_attention_modules(_stub_resolver)):
+ hardware_module._determine_attention_impl_for_gpu_estimate(config)
+ return captured["model_class"]
+
+ def test_raw_config_resolves_model_class_from_model_type(self):
+ from transformers import LlamaForCausalLM
+ model_class = self._resolved_model_class({"model_type": "llama", "hidden_size": 4096})
+ self.assertIs(model_class, LlamaForCausalLM)
+
+ def test_unknown_model_type_resolves_no_model_class(self):
+ self.assertIsNone(self._resolved_model_class({"model_type": "not_a_real_model"}))
+
+
class TestPerGpuFitGuardAllCounts(unittest.TestCase):
def test_training_estimate_resolves_attention_without_raising(self):
with (
diff --git a/studio/backend/tests/test_vram_estimation.py b/studio/backend/tests/test_vram_estimation.py
index f82885a7914..65c389c0efd 100644
--- a/studio/backend/tests/test_vram_estimation.py
+++ b/studio/backend/tests/test_vram_estimation.py
@@ -281,6 +281,35 @@ class TestExtractArchConfig(unittest.TestCase):
self.assertEqual(arch.quantization_skip_modules, ["model.layers.0.self_attn"])
self.assertEqual(arch.quant_4bit_factor, 3.6)
+ def test_quantization_fields_from_raw_config_json(self):
+ from utils.hardware import hardware as hardware_module
+
+ skip_modules = ["lm_head", "multi_modal_projector", "merger", "modality_projection"]
+ raw_config = {
+ "model_type": "gemma3",
+ "text_config": {
+ "model_type": "gemma3_text",
+ "hidden_size": 5376,
+ "num_hidden_layers": 62,
+ "num_attention_heads": 32,
+ "num_key_value_heads": 16,
+ "intermediate_size": 21504,
+ "vocab_size": 262208,
+ },
+ "quantization_config": {
+ "bnb_4bit_use_double_quant": True,
+ "llm_int8_skip_modules": skip_modules,
+ "quant_method": "bitsandbytes",
+ },
+ }
+ with patch("utils.transformers_version._load_config_json", return_value = raw_config):
+ config = hardware_module._load_config_for_gpu_estimate(
+ "unsloth/gemma-3-27b-it-bnb-4bit"
+ )
+ arch = extract_arch_config(config)
+ self.assertEqual(arch.quant_4bit_factor, 3.6)
+ self.assertEqual(arch.quantization_skip_modules, skip_modules)
+
class TestModelWeightsBytes(unittest.TestCase):
def test_llama_8b_fp16(self):
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index fb4a91241f5..01b236db31b 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -4852,9 +4852,14 @@ def _determine_attention_impl_for_gpu_estimate(config) -> str:
from unsloth.models._utils import resolve_attention_implementation
from transformers import AutoModel, AutoModelForCausalLM
+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING
# resolve_attention_implementation writes _attn_implementation onto the config and propagates to nested sub-configs, so a shallow copy would still mutate the cached config's shared inner objects.
config_copy = copy.deepcopy(config)
+ model_type = getattr(config_copy, "model_type", None)
+ config_class = (
+ CONFIG_MAPPING[model_type] if model_type in CONFIG_MAPPING else config_copy.__class__
+ )
model_class = None
for auto_model in (AutoModelForCausalLM, AutoModel):
@@ -4862,8 +4867,8 @@ def _determine_attention_impl_for_gpu_estimate(config) -> str:
if mapping is None:
continue
try:
- if config_copy.__class__ in mapping:
- model_class = mapping[config_copy.__class__]
+ if config_class in mapping:
+ model_class = mapping[config_class]
break
except Exception:
continue
diff --git a/studio/backend/utils/hardware/vram_estimation.py b/studio/backend/utils/hardware/vram_estimation.py
index e97462e1d1e..e7ceac5dded 100644
--- a/studio/backend/utils/hardware/vram_estimation.py
+++ b/studio/backend/utils/hardware/vram_estimation.py
@@ -216,7 +216,8 @@ def extract_arch_config(hf_config) -> Optional[ModelArchConfig]:
text_config = getattr(hf_config, "text_config", None) or hf_config
quantization_config = getattr(hf_config, "quantization_config", None) or {}
if not isinstance(quantization_config, dict):
- quantization_config = getattr(quantization_config, "to_dict", lambda: {})()
+ to_dict = getattr(quantization_config, "to_dict", None)
+ quantization_config = to_dict() if to_dict else getattr(quantization_config, "__dict__", {})
quant_4bit_factor = (
DOUBLE_QUANT_4BIT_FACTOR
if quantization_config.get("bnb_4bit_use_double_quant", False)