LLM inference in C/C++
C++ ggml
convert : speed up model endian conversion and add more datatype support (#30647)
Aaron Teob84ea5cf5e56
diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.pyindex 31618acfc7..3b20998c68 100755--- a/gguf-py/gguf/scripts/gguf_convert_endian.py+++ b/gguf-py/gguf/scripts/gguf_convert_endian.py@@ -19,71 +19,27 @@ import gguflogger = logging.getLogger("gguf-convert-endian")-def byteswap_noop(tensor, block_offs):- # this function is used when byteswapping is not needed- pass---def byteswap_q4_0(tensor, block_offs):- # Each block_q4_0 consists of an f16 delta (scaling factor) followed by 16 int8 quantizations.-- # Byte-Swap f16 sized delta field- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)- delta.byteswap(inplace=True)---def byteswap_q8_0(tensor, block_offs):- # Each block_q8_0 consists of an f16 delta (scaling factor) followed by 32 int8 quantizations.-- # Byte-Swap f16 sized delta field- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)- delta.byteswap(inplace=True)---def byteswap_q4_k(tensor, block_offs):- # Each block_q4_k consists of 2 f16 values followed by 140 int8 values.-- # Byte-Swap f16 sized fields- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)- delta.byteswap(inplace=True)-- delta = tensor.data[block_offs + 2:block_offs + 4].view(dtype=np.uint16)- delta.byteswap(inplace=True)---def byteswap_q6_k(tensor, block_offs):- # Each block_q6_k consists of 208 int8 values followed by 1 f16 value.-- # Byte-Swap f16 sized field- delta = tensor.data[block_offs + 208:block_offs + 210].view(dtype=np.uint16)- delta.byteswap(inplace=True)---def byteswap_q1_0(tensor, block_offs):- # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.-- # Byte-Swap f16 sized delta field- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)- delta.byteswap(inplace=True)---def byteswap_tq2_0(tensor, block_offs):- # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.-- # Byte-Swap f16 sized field- delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)- delta.byteswap(inplace=True)---byteswap_tensors = {- gguf.GGMLQuantizationType.Q1_0: byteswap_q1_0,- gguf.GGMLQuantizationType.Q4_0: byteswap_q4_0,- gguf.GGMLQuantizationType.Q8_0: byteswap_q8_0,- gguf.GGMLQuantizationType.Q4_K: byteswap_q4_k,- gguf.GGMLQuantizationType.Q6_K: byteswap_q6_k,- gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,- gguf.GGMLQuantizationType.MXFP4: byteswap_noop,- gguf.GGMLQuantizationType.NVFP4: byteswap_noop,+# byte offset and type of the multi-byte fields in each block, all other fields are single bytes+# use the width ggml reads the field with, e.g. Q5_0 qh is uint8_t[4] but read as uint32_t+byteswap_tensors: dict[gguf.GGMLQuantizationType, tuple[tuple[int, type[np.unsignedinteger]], ...]] = {+ gguf.GGMLQuantizationType.Q1_0: ((0, np.uint16),), # d+ gguf.GGMLQuantizationType.Q2_0: ((0, np.uint16),), # d+ gguf.GGMLQuantizationType.Q4_0: ((0, np.uint16),), # d+ gguf.GGMLQuantizationType.Q4_1: ((0, np.uint16), (2, np.uint16)), # d, m+ gguf.GGMLQuantizationType.Q5_0: ((0, np.uint16), (2, np.uint32)), # d, qh+ gguf.GGMLQuantizationType.Q5_1: ((0, np.uint16), (2, np.uint16), (4, np.uint32)), # d, m, qh+ gguf.GGMLQuantizationType.Q8_0: ((0, np.uint16),), # d+ gguf.GGMLQuantizationType.Q2_K: ((80, np.uint16), (82, np.uint16)), # d, dmin+ gguf.GGMLQuantizationType.Q3_K: ((108, np.uint16),), # d+ gguf.GGMLQuantizationType.Q4_K: ((0, np.uint16), (2, np.uint16)), # d, dmin+ gguf.GGMLQuantizationType.Q5_K: ((0, np.uint16), (2, np.uint16)), # d, dmin+ gguf.GGMLQuantizationType.Q6_K: ((208, np.uint16),), # d+ gguf.GGMLQuantizationType.IQ4_NL: ((0, np.uint16),), # d+ gguf.GGMLQuantizationType.IQ4_XS: ((0, np.uint16), (2, np.uint16)), # d, scales_h+ gguf.GGMLQuantizationType.TQ1_0: ((52, np.uint16),), # d+ gguf.GGMLQuantizationType.TQ2_0: ((64, np.uint16),), # d+ gguf.GGMLQuantizationType.MXFP4: (), # noop+ gguf.GGMLQuantizationType.NVFP4: (), # noop}@@ -140,28 +96,12 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None# Byte-swap tensor data if necessaryif tensor.tensor_type in byteswap_tensors:- # first flatten structure- oldshape = tensor.data.shape- newshape = 1- for i in tensor.data.shape:- newshape *= i-- tensor.data.resize(newshape)-- block_size = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]- byteswap_func = byteswap_tensors[tensor.tensor_type]-- n_blocks = len(tensor.data) // block_size- for block_num in (inner_pbar := tqdm(range(n_blocks), desc="Byte-swapping Blocks", leave=False)):- block_offs = block_num * block_size-- byteswap_func(tensor, block_offs)-- if block_num % 100000 == 0:- inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]")-- # restore old shape in case it's ever used- tensor.data.resize(oldshape)+ # one row per block, then swap each field of all blocks at once+ type_size = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]+ blocks = tensor.data.reshape(-1, type_size, copy=False)+ for offs, dtype in byteswap_tensors[tensor.tensor_type]:+ size = np.dtype(dtype).itemsize+ blocks[:, offs:offs + size].view(dtype=dtype)[:, 0].byteswap(inplace=True)elif tensor.tensor_type == gguf.GGMLQuantizationType.BF16:# Special case for BF16# It is 2-bytes data, but by default view loads it as 1-byte data.@@ -173,6 +113,7 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> Nonepbar.set_description(log_message)+ reader.data.flush()logger.info("* Completion")