Skip to content

LLM inference in C/C++

C++ ggml

convert: Add endianness conversion for Q1 and TQ2 quantizations (#26618)

Andreas Krebbel020760adfc93
diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.py
index 164c9171e0..31618acfc7 100755
--- a/gguf-py/gguf/scripts/gguf_convert_endian.py
+++ b/gguf-py/gguf/scripts/gguf_convert_endian.py
@@ -59,11 +59,29 @@ def byteswap_q6_k(tensor, block_offs):
delta.byteswap(inplace=True)
+def byteswap_q1_0(tensor, block_offs):
+ # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.
+
+ # Byte-Swap f16 sized delta field
+ delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
+ delta.byteswap(inplace=True)
+
+
+def byteswap_tq2_0(tensor, block_offs):
+ # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.
+
+ # Byte-Swap f16 sized field
+ delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)
+ delta.byteswap(inplace=True)
+
+
byteswap_tensors = {
+ gguf.GGMLQuantizationType.Q1_0: byteswap_q1_0,
gguf.GGMLQuantizationType.Q4_0: byteswap_q4_0,
gguf.GGMLQuantizationType.Q8_0: byteswap_q8_0,
gguf.GGMLQuantizationType.Q4_K: byteswap_q4_k,
gguf.GGMLQuantizationType.Q6_K: byteswap_q6_k,
+ gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,
gguf.GGMLQuantizationType.MXFP4: byteswap_noop,
gguf.GGMLQuantizationType.NVFP4: byteswap_noop,
}