Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
tools/gguf_reader.py: bound the header reads and report malformed metadata as ValueError (#2288, #1919)
Keyan Hu409f2466ad2a
diff --git a/tools/gguf_reader.py b/tools/gguf_reader.pyindex 4da9d2d24..de30d3b92 100644--- a/tools/gguf_reader.py+++ b/tools/gguf_reader.py@@ -10,6 +10,7 @@ no ggml dependency)"). It parses the header only; tensor data is never read.from __future__ import annotationsimport dataclasses+import osimport pathlibimport structfrom typing import Any@@ -77,6 +78,10 @@ class TensorInfo:return self.elements // block_elems * block_bytes+_META_FMT = {"u8": "B", "i8": "b", "u16": "H", "i16": "h", "u32": "I", "i32": "i",+ "f32": "f", "bool": "?", "u64": "Q", "i64": "q", "f64": "d"}++class GGUFFile:def __init__(self, path: pathlib.Path):self.path = pathlib.Path(path)@@ -85,55 +90,82 @@ class GGUFFile:self.version = 0self.alignment = 32with self.path.open("rb") as fh:- self._parse(fh)+ self._size = os.fstat(fh.fileno()).st_size+ self._pos = 0+ try:+ self._parse(fh)+ except ValueError as e:+ raise ValueError(f"{self.path.name}: {e}") from Noneself.data_start = self._data_start# ------------------------------------------------------------------ internals+ def _read(self, fh, n: int) -> bytes:+ left = self._size - self._pos+ if n < 0 or n > left:+ raise ValueError(f"header ends early: {n:,} bytes needed at offset {self._pos:,}, {left:,} left")+ raw = fh.read(n)+ self._pos += len(raw)+ if len(raw) != n:+ raise ValueError(f"header ends early at offset {self._pos:,}")+ return raw++ def _unpack(self, fh, fmt: str) -> tuple:+ return struct.unpack(fmt, self._read(fh, struct.calcsize(fmt)))++ def _count(self, fh, count: int, min_bytes: int, what: str) -> int:+ if count * min_bytes > self._size - self._pos:+ raise ValueError(f"{count:,} {what} cannot fit in the {self._size - self._pos:,} bytes left")+ return count+def _parse(self, fh) -> None:- magic = struct.unpack("<I", fh.read(4))[0]+ (magic,) = self._unpack(fh, "<I")if magic != GGUF_MAGIC:- raise ValueError(f"{self.path.name}: not a GGUF file (magic {magic:#x})")- self.version, n_tensors, n_kv = struct.unpack("<IQQ", fh.read(20))+ raise ValueError(f"not a GGUF file (magic {magic:#x})")+ self.version, n_tensors, n_kv = self._unpack(fh, "<IQQ")if self.version != 3:- raise ValueError(f"{self.path.name}: GGUF v{self.version}, this reader handles v3")- for _ in range(n_kv):+ raise ValueError(f"GGUF v{self.version}, this reader handles v3")+ for _ in range(self._count(fh, n_kv, 12, "metadata entries")): # key length (8) + value type (4)key = self._str(fh)self.metadata[key] = self._value(fh)- for _ in range(n_tensors):+ for _ in range(self._count(fh, n_tensors, 24, "tensors")): # name length, n_dims, type, offsetname = self._str(fh)- (n_dims,) = struct.unpack("<I", fh.read(4))- shape = list(struct.unpack(f"<{n_dims}Q", fh.read(8 * n_dims)))- type_id, offset = struct.unpack("<IQ", fh.read(12))+ (n_dims,) = self._unpack(fh, "<I")+ shape = list(self._unpack(fh, f"<{self._count(fh, n_dims, 8, 'dimensions')}Q"))+ type_id, offset = self._unpack(fh, "<IQ")self.tensors.append(TensorInfo(name, shape, type_id,GGML_TYPES.get(type_id, f"type{type_id}"), offset))align = self.metadata.get("general.alignment")- if isinstance(align, int) and align:+ if align is not None:+ if type(align) is not int or align <= 0 or align % 8:+ raise ValueError("general.alignment must be a positive multiple of 8")self.alignment = align- pos = fh.tell()+ pos = self._posself._data_start = (pos + self.alignment - 1) // self.alignment * self.alignmentdef _str(self, fh) -> str:- (n,) = struct.unpack("<Q", fh.read(8))- return fh.read(n).decode("utf-8", "replace")+ (n,) = self._unpack(fh, "<Q")+ return self._read(fh, n).decode("utf-8", "replace")++ def _meta_type(self, t: int) -> tuple[str, int | None]:+ if t not in GGUF_META:+ raise ValueError(f"unknown metadata value type {t}")+ return GGUF_META[t]def _value(self, fh):- (t,) = struct.unpack("<I", fh.read(4))- name, size = GGUF_META[t]+ (t,) = self._unpack(fh, "<I")+ name, size = self._meta_type(t)if name == "string":return self._str(fh)if name == "array":- (et,) = struct.unpack("<I", fh.read(4))- (count,) = struct.unpack("<Q", fh.read(8))- ename, esize = GGUF_META[et]+ et, count = self._unpack(fh, "<IQ")+ ename, esize = self._meta_type(et)+ if ename == "array":+ raise ValueError("nested metadata arrays are not supported")if ename == "string":- return [self._str(fh) for _ in range(count)]- fmt = {"u8": "B", "i8": "b", "u16": "H", "i16": "h", "u32": "I", "i32": "i",- "f32": "f", "bool": "?", "u64": "Q", "i64": "q", "f64": "d"}[ename]- raw = fh.read(esize * count)- return list(struct.unpack(f"<{count}{fmt}", raw))- fmt = {"u8": "B", "i8": "b", "u16": "H", "i16": "h", "u32": "I", "i32": "i",- "f32": "f", "bool": "?", "u64": "Q", "i64": "q", "f64": "d"}[name]- return struct.unpack(f"<{fmt}", fh.read(size))[0]+ return [self._str(fh) for _ in range(self._count(fh, count, 8, "array strings"))]+ self._count(fh, count, esize, "array elements")+ return list(self._unpack(fh, f"<{count}{_META_FMT[ename]}"))+ return self._unpack(fh, f"<{_META_FMT[name]}")[0]# ------------------------------------------------------------------ helpersdef by_type(self) -> dict[str, int]:diff --git a/tools/test_gguf_reader_malformed.py b/tools/test_gguf_reader_malformed.pynew file mode 100644index 000000000..98358f029--- /dev/null+++ b/tools/test_gguf_reader_malformed.py@@ -0,0 +1,176 @@+"""A malformed GGUF header is refused with a ValueError that names the file, never a KeyError, OverflowError+or struct.error (#1919). Tensor-size support and setup compatibility policy are unchanged.+Over minimal GGUF v3 files written here (no download, no model, no GPU).++ python -m unittest tools.test_gguf_reader_malformed+"""+from __future__ import annotations++import contextlib+import io+import struct+import sys+import tempfile+import unittest+from unittest import mock+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT / "tools"))+sys.path.insert(0, str(ROOT))+import setup as S # noqa: E402+from gguf_reader import GGUFFile # noqa: E402+++class Stop(Exception):+ """setup.fail(), caught instead of exiting."""+++def gstr(s: bytes) -> bytes:+ return struct.pack("<Q", len(s)) + s+++def header(n_tensors: int, kv: bytes = b"", n_kv: int = 0) -> bytes:+ return struct.pack("<IIQQ", 0x46554747, 3, n_tensors, n_kv) + kv+++def tensor(name: bytes, n: int, type_id: int, offset: int) -> bytes:+ return gstr(name) + struct.pack("<IQIQ", 1, n, type_id, offset)+++def pad(b: bytes, data: int) -> bytes:+ return b + bytes(-len(b) % 32 + data)+++class MalformedHeader(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.path = Path(self.tmp.name) / "m-00001-of-00002.gguf"++ def tearDown(self):+ self.tmp.cleanup()++ def refused(self, raw: bytes) -> str:+ self.path.write_bytes(raw)+ with self.assertRaises(ValueError) as cm:+ GGUFFile(self.path)+ self.assertIn(self.path.name, str(cm.exception))+ return str(cm.exception)++ def test_unknown_metadata_value_type(self):+ self.refused(pad(header(0, gstr(b"k") + struct.pack("<I", 99), 1), 0))++ def test_unknown_array_element_type(self):+ self.refused(pad(header(0, gstr(b"k") + struct.pack("<IIQ", 9, 99, 1), 1), 0))++ def test_string_length_past_the_end(self):+ self.refused(pad(header(0, struct.pack("<Q", 2 ** 63), 1), 0))++ def test_array_count_past_the_end(self):+ self.refused(pad(header(0, gstr(b"k") + struct.pack("<IIQ", 9, 4, 2 ** 40), 1), 0))++ def test_string_array_count_past_the_end(self):+ self.refused(pad(header(0, gstr(b"k") + struct.pack("<IIQ", 9, 8, 2 ** 62), 1), 0))++ def test_tensor_count_past_the_end(self):+ self.refused(header(2 ** 40))++ def test_header_cut_mid_tensor_directory(self):+ self.refused(header(1) + tensor(b"blk.0.w", 8, 0, 0)[:-5])++ def test_header_cut_inside_the_magic(self):+ self.refused(b"GG")+++class Alignment(unittest.TestCase):+ setUp = MalformedHeader.setUp+ tearDown = MalformedHeader.tearDown+ refused = MalformedHeader.refused+ def test_invalid_alignment(self):+ for alignment in (0, -8, 3):+ with self.subTest(alignment=alignment):+ kv = gstr(b"general.alignment") + struct.pack("<Ii", 5, alignment)+ self.assertIn("alignment", self.refused(pad(header(0, kv, 1), 0)))+++class UnknownTensorCompatibility(unittest.TestCase):+ def test_readable_unknown_tensor_keeps_setup_policy(self):+ with tempfile.TemporaryDirectory() as tmp:+ p = Path(tmp) / "future.gguf"+ raw = pad(header(2) + tensor(b"known", 8, 0, 0) + tensor(b"future", 64, 999, 32), 256)+ p.write_bytes(raw)+ g = GGUFFile(p)+ self.assertIsNone(g.tensors[1].expected_bytes())+ self.assertTrue(S.whole_shard(p))+ S.check_shards([p])+ self.assertEqual(p.read_bytes(), raw)+++class ReaderCompatibility(unittest.TestCase):+ setUp = MalformedHeader.setUp+ tearDown = MalformedHeader.tearDown+ refused = MalformedHeader.refused++ def test_supported_scalar_and_array_values(self):+ values = [(0, 'B', 255), (1, 'b', -7), (2, 'H', 65535), (3, 'h', -300),+ (4, 'I', 2**31), (5, 'i', -70000), (6, 'f', 1.25), (7, '?', True),+ (10, 'Q', 2**63), (11, 'q', -2**40), (12, 'd', 1.125)]+ kv, expected = b'', {}+ for type_id, fmt, value in values:+ for array in (False, True):+ key = f'{type_id}_{array}'+ data = struct.pack('<' + fmt, value)+ kv += gstr(key.encode()) + (struct.pack('<IIQ', 9, type_id, 1) if array+ else struct.pack('<I', type_id)) + data+ expected[key] = [value] if array else value+ for key, data, value in [('text', struct.pack('<I', 8) + gstr('测试'.encode()), '测试'),+ ('strings', struct.pack('<IIQ', 9, 8, 2) + gstr(b'') + gstr(b'x'), ['', 'x']),+ ('empty', struct.pack('<IIQ', 9, 4, 0), [])]:+ kv += gstr(key.encode()) + data+ expected[key] = value+ self.path.write_bytes(pad(header(1, kv, len(expected)) + tensor(b'w', 8, 0, 0), 32))+ g = GGUFFile(self.path)+ self.assertEqual(g.metadata, expected)+ self.assertEqual((g.tensors[0].name, g.tensors[0].shape, g.tensors[0].expected_bytes()), ('w', [8], 32))+ S.check_shards([self.path])++ def test_metadata_only_file_needs_no_alignment_padding(self):+ raw = header(0, gstr(b'general.name') + struct.pack('<I', 8) + gstr(b't'), 1)+ self.path.write_bytes(raw)+ g = GGUFFile(self.path)+ self.assertEqual(g.metadata, {'general.name': 't'})+ self.assertEqual(g.tensors, [])+ self.assertGreater(g.data_start, len(raw))+ S.check_shards([self.path])+ self.assertEqual(self.path.read_bytes(), raw)++ def test_explicit_alignment_keeps_tensor_offsets(self):+ for alignment in (24, 64):+ with self.subTest(alignment=alignment):+ kv = gstr(b'general.alignment') + struct.pack('<II', 4, alignment)+ raw = header(1, kv, 1) + tensor(b'w', 8, 0, 0)+ start = (len(raw) + alignment - 1) // alignment * alignment+ self.path.write_bytes(raw + bytes(start - len(raw) + 32))+ g = GGUFFile(self.path)+ self.assertEqual((g.alignment, g.data_start), (alignment, start))+ S.check_shards([self.path])++ def test_setup_reports_malformed_file_without_changing_it(self):+ for raw in (header(0, gstr(b'k') + struct.pack('<I', 99), 1),+ header(0, struct.pack('<Q', 2**63) + bytes(4), 1)):+ with self.subTest(raw=raw.hex()):+ self.path.write_bytes(raw)+ self.assertFalse(S.whole_shard(self.path))+ out = io.StringIO()+ with contextlib.redirect_stdout(out), self.assertRaises(SystemExit):+ S.check_shards([self.path])+ self.assertIn(self.path.name + ' is not a whole GGUF shard', out.getvalue())+ self.assertEqual(self.path.read_bytes(), raw)++ def test_unsupported_array_and_impossible_dimension_count_are_named(self):+ self.refused(header(0, gstr(b'k') + struct.pack('<IIQ', 9, 9, 0), 1))+ self.refused(header(1) + gstr(b'w') + struct.pack('<I', 2**32 - 1) + bytes(12))+++if __name__ == "__main__":+ unittest.main()