LLM inference in C/C++
C++ ggml
convert: Add endianness conversion for Q1 and TQ2 quantizations (#26618)
Andreas Krebbel020760adfc93
diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.pyindex 164c9171e0..31618acfc7 100755--- a/gguf-py/gguf/scripts/gguf_convert_endian.py+++ b/gguf-py/gguf/scripts/gguf_convert_endian.py@@ -59,11 +59,29 @@ def byteswap_q6_k(tensor, block_offs):delta.byteswap(inplace=True)+def byteswap_q1_0(tensor, block_offs):+ # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.++ # Byte-Swap f16 sized delta field+ delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)+ delta.byteswap(inplace=True)+++def byteswap_tq2_0(tensor, block_offs):+ # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.++ # Byte-Swap f16 sized field+ delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)+ delta.byteswap(inplace=True)++byteswap_tensors = {+ gguf.GGMLQuantizationType.Q1_0: byteswap_q1_0,gguf.GGMLQuantizationType.Q4_0: byteswap_q4_0,gguf.GGMLQuantizationType.Q8_0: byteswap_q8_0,gguf.GGMLQuantizationType.Q4_K: byteswap_q4_k,gguf.GGMLQuantizationType.Q6_K: byteswap_q6_k,+ gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,gguf.GGMLQuantizationType.MXFP4: byteswap_noop,gguf.GGMLQuantizationType.NVFP4: byteswap_noop,}