Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: load GGUF models faster by skipping the vocabulary read (#14573)
Nilay79db5ad81048
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.pyindex 9d36726fa61..cd1cfc8ed67 100644--- a/studio/backend/core/inference/offload_layout.py+++ b/studio/backend/core/inference/offload_layout.py@@ -152,14 +152,39 @@ def layout_from_gguf(path: str, *, all_shards: bool = False) -> ModelLayout:``all_shards`` reads every sibling shard; all of them must be present."""try:- from gguf import GGUFReader- readers = [GGUFReader(path)]+ import struct++ import numpy as np+ from gguf import GGUFEndian, GGUFReader, GGUFValueType++ class _LayoutReader(GGUFReader):+ # GGUFReader makes a numpy view per array element, seconds for a vocabulary; no string array is read here.+ def _get_field_parts(self, orig_offs, raw_type):+ if raw_type != GGUFValueType.ARRAY:+ return super()._get_field_parts(orig_offs, raw_type)+ raw_itype = self._get(orig_offs, np.uint32)+ alen = self._get(orig_offs + 4, np.uint64)+ itype = GGUFValueType(raw_itype[0])+ types = [GGUFValueType.ARRAY, itype]+ nptype = self.gguf_scalar_to_np.get(itype)+ if nptype is not None:+ items = self._get(orig_offs + 12, nptype, alen[0])+ return 12 + int(items.nbytes), [raw_itype, alen, items], [2], types+ if itype != GGUFValueType.STRING:+ return super()._get_field_parts(orig_offs, raw_type)+ length = struct.Struct("<Q" if self.endianess == GGUFEndian.LITTLE else ">Q")+ offs = orig_offs + 12+ for _ in range(int(alen[0])):+ offs += 8 + length.unpack_from(self.data, offs)[0]+ return offs - orig_offs, [raw_itype, alen], [], types++ readers = [_LayoutReader(path)]if all_shards and int(_field(readers[0], "split.count") or 0) > 1:shards = split_shard_paths(path)if not shards or not all(os.path.isfile(p) for p in shards):logger.debug("offload layout: split %s is missing a shard", path)return ModelLayout()- readers = [GGUFReader(p) for p in shards]+ readers = [_LayoutReader(p) for p in shards]except Exception as exc:logger.debug("offload layout: cannot read %s (%s)", path, exc)return ModelLayout()diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.pyindex 792b020eedb..79958292fd5 100644--- a/studio/backend/tests/test_offload_planner.py+++ b/studio/backend/tests/test_offload_planner.py@@ -1013,6 +1013,47 @@ def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):assert layout.token_embd_bytes == 4 * (1024 + 4096)[email protected]("endianess", ["LITTLE", "BIG"])+def test_reading_the_layout_does_not_walk_the_vocabulary(tmp_path, monkeypatch, endianess):+ import numpy as np+ from gguf import GGUFEndian, GGUFReader, GGUFWriter++ vocab = 50_000+ path = tmp_path / "vocab.gguf"+ writer = GGUFWriter(str(path), "nemotron_h", endianess = GGUFEndian[endianess])+ writer.add_block_count(3)+ writer.add_head_count(8)+ writer.add_key_length(128)+ writer.add_value_length(128)+ # A per-layer array, so the numeric-array path must keep every element.+ writer.add_array("nemotron_h.attention.head_count_kv", [0, 4, 2])+ writer.add_token_list([f"tok{i}" for i in range(vocab)])+ writer.add_token_scores([0.0] * vocab)+ writer.add_token_types([1] * vocab)+ writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))+ writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))+ writer.write_header_to_file()+ writer.write_kv_data_to_file()+ writer.write_tensors_to_file()+ writer.close()++ reads = 0+ real_get = GGUFReader._get++ def counting_get(self, *args, **kwargs):+ nonlocal reads+ reads += 1+ return real_get(self, *args, **kwargs)++ monkeypatch.setattr(GGUFReader, "_get", counting_get)+ layout = layout_from_gguf(str(path))+ assert layout.complete+ assert layout.token_embd_bytes == 4 * 1024+ assert layout.n_attention_layers == 2+ assert layout.kv_bytes_per_token_f16 == (4 + 2) * (128 + 128) * 2+ assert reads < 1000++def test_token_embd_norm_is_not_an_input_embedding():"""token_embd_norm follows the repeating layers rather than the input layer."""tensors = _shard_tensors(range(64)) + [@@ -1567,19 +1608,23 @@ def test_split_shard_paths_follow_llama_cpp_naming():assert split_shard_paths("/m/model-Q4.gguf") is None-def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path, monkeypatch):- import gguf+def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path):+ import numpy as np+ from gguf import GGUFWritershards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]- for shard in shards:- shard.write_bytes(b"GGUF")- by_path = {- str(shards[0]): _StubReader(_shard_fields(**{"split.count": 2}), _shard_tensors(range(32))),- str(shards[1]): _StubReader(- _shard_fields(**{"split.count": 2}), _shard_tensors(range(32, 64))- ),- }- monkeypatch.setattr(gguf, "GGUFReader", lambda path: by_path[str(path)])+ for shard, blocks in zip(shards, (range(32), range(32, 64))):+ writer = GGUFWriter(str(shard), "llama")+ for key, value in _shard_fields().items():+ if key.startswith("llama."):+ writer.add_uint32(key, value)+ writer.add_uint16("split.count", 2)+ for tensor in _shard_tensors(blocks):+ writer.add_tensor(tensor.name, np.zeros(4, dtype = np.float32))+ writer.write_header_to_file()+ writer.write_kv_data_to_file()+ writer.write_tensors_to_file()+ writer.close()assert not layout_from_gguf(str(shards[0])).completewhole = layout_from_gguf(str(shards[0]), all_shards = True)