Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load GGUF models faster by skipping the vocabulary read (#14573)

Nilay79db5ad81048
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.py
index 9d36726fa61..cd1cfc8ed67 100644
--- a/studio/backend/core/inference/offload_layout.py
+++ b/studio/backend/core/inference/offload_layout.py
@@ -152,14 +152,39 @@ def layout_from_gguf(path: str, *, all_shards: bool = False) -> ModelLayout:
``all_shards`` reads every sibling shard; all of them must be present.
"""
try:
- from gguf import GGUFReader
- readers = [GGUFReader(path)]
+ import struct
+
+ import numpy as np
+ from gguf import GGUFEndian, GGUFReader, GGUFValueType
+
+ class _LayoutReader(GGUFReader):
+ # GGUFReader makes a numpy view per array element, seconds for a vocabulary; no string array is read here.
+ def _get_field_parts(self, orig_offs, raw_type):
+ if raw_type != GGUFValueType.ARRAY:
+ return super()._get_field_parts(orig_offs, raw_type)
+ raw_itype = self._get(orig_offs, np.uint32)
+ alen = self._get(orig_offs + 4, np.uint64)
+ itype = GGUFValueType(raw_itype[0])
+ types = [GGUFValueType.ARRAY, itype]
+ nptype = self.gguf_scalar_to_np.get(itype)
+ if nptype is not None:
+ items = self._get(orig_offs + 12, nptype, alen[0])
+ return 12 + int(items.nbytes), [raw_itype, alen, items], [2], types
+ if itype != GGUFValueType.STRING:
+ return super()._get_field_parts(orig_offs, raw_type)
+ length = struct.Struct("<Q" if self.endianess == GGUFEndian.LITTLE else ">Q")
+ offs = orig_offs + 12
+ for _ in range(int(alen[0])):
+ offs += 8 + length.unpack_from(self.data, offs)[0]
+ return offs - orig_offs, [raw_itype, alen], [], types
+
+ readers = [_LayoutReader(path)]
if all_shards and int(_field(readers[0], "split.count") or 0) > 1:
shards = split_shard_paths(path)
if not shards or not all(os.path.isfile(p) for p in shards):
logger.debug("offload layout: split %s is missing a shard", path)
return ModelLayout()
- readers = [GGUFReader(p) for p in shards]
+ readers = [_LayoutReader(p) for p in shards]
except Exception as exc:
logger.debug("offload layout: cannot read %s (%s)", path, exc)
return ModelLayout()
diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.py
index 792b020eedb..79958292fd5 100644
--- a/studio/backend/tests/test_offload_planner.py
+++ b/studio/backend/tests/test_offload_planner.py
@@ -1013,6 +1013,47 @@ def test_the_layout_reader_totals_every_tensor_from_a_real_gguf(tmp_path):
assert layout.token_embd_bytes == 4 * (1024 + 4096)
[email protected]("endianess", ["LITTLE", "BIG"])
+def test_reading_the_layout_does_not_walk_the_vocabulary(tmp_path, monkeypatch, endianess):
+ import numpy as np
+ from gguf import GGUFEndian, GGUFReader, GGUFWriter
+
+ vocab = 50_000
+ path = tmp_path / "vocab.gguf"
+ writer = GGUFWriter(str(path), "nemotron_h", endianess = GGUFEndian[endianess])
+ writer.add_block_count(3)
+ writer.add_head_count(8)
+ writer.add_key_length(128)
+ writer.add_value_length(128)
+ # A per-layer array, so the numeric-array path must keep every element.
+ writer.add_array("nemotron_h.attention.head_count_kv", [0, 4, 2])
+ writer.add_token_list([f"tok{i}" for i in range(vocab)])
+ writer.add_token_scores([0.0] * vocab)
+ writer.add_token_types([1] * vocab)
+ writer.add_tensor("token_embd.weight", np.zeros(1024, dtype = np.float32))
+ writer.add_tensor("blk.0.attn_q.weight", np.zeros(256, dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+
+ reads = 0
+ real_get = GGUFReader._get
+
+ def counting_get(self, *args, **kwargs):
+ nonlocal reads
+ reads += 1
+ return real_get(self, *args, **kwargs)
+
+ monkeypatch.setattr(GGUFReader, "_get", counting_get)
+ layout = layout_from_gguf(str(path))
+ assert layout.complete
+ assert layout.token_embd_bytes == 4 * 1024
+ assert layout.n_attention_layers == 2
+ assert layout.kv_bytes_per_token_f16 == (4 + 2) * (128 + 128) * 2
+ assert reads < 1000
+
+
def test_token_embd_norm_is_not_an_input_embedding():
"""token_embd_norm follows the repeating layers rather than the input layer."""
tensors = _shard_tensors(range(64)) + [
@@ -1567,19 +1608,23 @@ def test_split_shard_paths_follow_llama_cpp_naming():
assert split_shard_paths("/m/model-Q4.gguf") is None
-def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path, monkeypatch):
- import gguf
+def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path):
+ import numpy as np
+ from gguf import GGUFWriter
shards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]
- for shard in shards:
- shard.write_bytes(b"GGUF")
- by_path = {
- str(shards[0]): _StubReader(_shard_fields(**{"split.count": 2}), _shard_tensors(range(32))),
- str(shards[1]): _StubReader(
- _shard_fields(**{"split.count": 2}), _shard_tensors(range(32, 64))
- ),
- }
- monkeypatch.setattr(gguf, "GGUFReader", lambda path: by_path[str(path)])
+ for shard, blocks in zip(shards, (range(32), range(32, 64))):
+ writer = GGUFWriter(str(shard), "llama")
+ for key, value in _shard_fields().items():
+ if key.startswith("llama."):
+ writer.add_uint32(key, value)
+ writer.add_uint16("split.count", 2)
+ for tensor in _shard_tensors(blocks):
+ writer.add_tensor(tensor.name, np.zeros(4, dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
assert not layout_from_gguf(str(shards[0])).complete
whole = layout_from_gguf(str(shards[0]), all_shards = True)