Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Size training VRAM with the model's attention kernel and 4-bit config (#10815)
Nilay6417d4b93577
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.pyindex 3b0d1aaac79..82a1c9a9aad 100644--- a/studio/backend/tests/test_gpu_selection.py+++ b/studio/backend/tests/test_gpu_selection.py@@ -1751,6 +1751,31 @@ class TestMinGpuVram(unittest.TestCase):self.assertEqual(breakdown.total, breakdown.min_gpu_vram(1))+class TestAttentionEstimateModelClass(unittest.TestCase):+ def _resolved_model_class(self, raw_config):+ from utils.hardware import hardware as hardware_module++ captured = {}++ def _stub_resolver(model_class, cfg):+ captured["model_class"] = model_class+ return "sdpa"++ with patch("utils.transformers_version._load_config_json", return_value = raw_config):+ config = hardware_module._load_config_for_gpu_estimate("unsloth/test")+ with patch.dict(sys.modules, _fake_unsloth_attention_modules(_stub_resolver)):+ hardware_module._determine_attention_impl_for_gpu_estimate(config)+ return captured["model_class"]++ def test_raw_config_resolves_model_class_from_model_type(self):+ from transformers import LlamaForCausalLM+ model_class = self._resolved_model_class({"model_type": "llama", "hidden_size": 4096})+ self.assertIs(model_class, LlamaForCausalLM)++ def test_unknown_model_type_resolves_no_model_class(self):+ self.assertIsNone(self._resolved_model_class({"model_type": "not_a_real_model"}))++class TestPerGpuFitGuardAllCounts(unittest.TestCase):def test_training_estimate_resolves_attention_without_raising(self):with (diff --git a/studio/backend/tests/test_vram_estimation.py b/studio/backend/tests/test_vram_estimation.pyindex f82885a7914..65c389c0efd 100644--- a/studio/backend/tests/test_vram_estimation.py+++ b/studio/backend/tests/test_vram_estimation.py@@ -281,6 +281,35 @@ class TestExtractArchConfig(unittest.TestCase):self.assertEqual(arch.quantization_skip_modules, ["model.layers.0.self_attn"])self.assertEqual(arch.quant_4bit_factor, 3.6)+ def test_quantization_fields_from_raw_config_json(self):+ from utils.hardware import hardware as hardware_module++ skip_modules = ["lm_head", "multi_modal_projector", "merger", "modality_projection"]+ raw_config = {+ "model_type": "gemma3",+ "text_config": {+ "model_type": "gemma3_text",+ "hidden_size": 5376,+ "num_hidden_layers": 62,+ "num_attention_heads": 32,+ "num_key_value_heads": 16,+ "intermediate_size": 21504,+ "vocab_size": 262208,+ },+ "quantization_config": {+ "bnb_4bit_use_double_quant": True,+ "llm_int8_skip_modules": skip_modules,+ "quant_method": "bitsandbytes",+ },+ }+ with patch("utils.transformers_version._load_config_json", return_value = raw_config):+ config = hardware_module._load_config_for_gpu_estimate(+ "unsloth/gemma-3-27b-it-bnb-4bit"+ )+ arch = extract_arch_config(config)+ self.assertEqual(arch.quant_4bit_factor, 3.6)+ self.assertEqual(arch.quantization_skip_modules, skip_modules)+class TestModelWeightsBytes(unittest.TestCase):def test_llama_8b_fp16(self):diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex fb4a91241f5..01b236db31b 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -4852,9 +4852,14 @@ def _determine_attention_impl_for_gpu_estimate(config) -> str:from unsloth.models._utils import resolve_attention_implementationfrom transformers import AutoModel, AutoModelForCausalLM+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING# resolve_attention_implementation writes _attn_implementation onto the config and propagates to nested sub-configs, so a shallow copy would still mutate the cached config's shared inner objects.config_copy = copy.deepcopy(config)+ model_type = getattr(config_copy, "model_type", None)+ config_class = (+ CONFIG_MAPPING[model_type] if model_type in CONFIG_MAPPING else config_copy.__class__+ )model_class = Nonefor auto_model in (AutoModelForCausalLM, AutoModel):@@ -4862,8 +4867,8 @@ def _determine_attention_impl_for_gpu_estimate(config) -> str:if mapping is None:continuetry:- if config_copy.__class__ in mapping:- model_class = mapping[config_copy.__class__]+ if config_class in mapping:+ model_class = mapping[config_class]breakexcept Exception:continuediff --git a/studio/backend/utils/hardware/vram_estimation.py b/studio/backend/utils/hardware/vram_estimation.pyindex e97462e1d1e..e7ceac5dded 100644--- a/studio/backend/utils/hardware/vram_estimation.py+++ b/studio/backend/utils/hardware/vram_estimation.py@@ -216,7 +216,8 @@ def extract_arch_config(hf_config) -> Optional[ModelArchConfig]:text_config = getattr(hf_config, "text_config", None) or hf_configquantization_config = getattr(hf_config, "quantization_config", None) or {}if not isinstance(quantization_config, dict):- quantization_config = getattr(quantization_config, "to_dict", lambda: {})()+ to_dict = getattr(quantization_config, "to_dict", None)+ quantization_config = to_dict() if to_dict else getattr(quantization_config, "__dict__", {})quant_4bit_factor = (DOUBLE_QUANT_4BIT_FACTORif quantization_config.get("bnb_4bit_use_double_quant", False)