Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
11.8 KB
Raw Blame
#!/usr/bin/env python3
"""Dequantization of GGUF/ggml tensor blocks to float32, pure numpy.

Ported block-for-block from ggml-org/ggml src/ggml-quants.c (dequantize_row_*),
using the block layouts in src/ggml-common.h. This module owns the numerical
decode used by the converter profiles; gguf_reader.py owns structure only.

Only the types listed in _DEQUANTIZERS are implemented. dequantize() raises
GGUFError for any other type, so a profile never silently mis-decodes bytes.
Correctness is pinned by tests/test_gguf_dequant.py, which compares against
llama.cpp's gguf-py reference on random blocks and against committed vectors.
"""

import numpy as np

from gguf_reader import (
    GGML_TYPE_BF16,
    GGML_TYPE_F16,
    GGML_TYPE_F32,
    GGML_TYPE_Q2_K,
    GGML_TYPE_Q3_K,
    GGML_TYPE_Q4_K,
    GGML_TYPE_Q5_K,
    GGML_TYPE_Q6_K,
    GGML_TYPE_Q8_0,
    GGUFError,
    ggml_block_spec,
    ggml_type_name,
)

QK_K = 256
_MASK_KMASK1 = 0x03030303
_MASK_KMASK2 = 0x0F0F0F0F


def _f16_to_f32(pair_bytes):
    """(n, 2) uint8 or (n,) uint16 -> float32."""
    u16 = pair_bytes.view("<u2").reshape(-1)
    return u16.view(np.float16).astype(np.float32)


def _bf16_to_f32(word_bytes):
    u16 = word_bytes.view("<u2").reshape(-1).astype(np.uint32)
    return (u16 << 16).view(np.float32)


def _unpack_scale_min_k4(sc_bytes):
    """Sixteen 6-bit (scale, min) pairs packed into 12 bytes (get_scale_min_k4).

    ggml-quants.c get_scale_min_k4(): the first four pairs use 6 bits directly
    (scale in the low 6 bits, min in the high 6 bits of a sibling byte), the
    remaining pairs are split across three bytes by a bit shuffle.
    """
    num_blocks = sc_bytes.shape[0]
    scale = np.empty((num_blocks, 8), dtype=np.float32)
    minimum = np.empty((num_blocks, 8), dtype=np.float32)
    scale[:, 0:4] = sc_bytes[:, 0:4] & 63
    minimum[:, 0:4] = sc_bytes[:, 4:8] & 63
    scale[:, 4:8] = (sc_bytes[:, 8:12] & 0x0F) | ((sc_bytes[:, 0:4] >> 6) << 4)
    minimum[:, 4:8] = (sc_bytes[:, 8:12] >> 4) | ((sc_bytes[:, 4:8] >> 6) << 4)
    return scale, minimum


def dequantize_q4_K(blocks):
    # Q4_K (ggml-common.h block_q4_K, 144 bytes/256 elems):
    #   d/dmin (fp16), scales[12] (sixteen 6-bit scale/min pairs),
    #   qs[128] 4-bit packed quants. 8 sub-blocks of 32 values each.
    num_blocks = blocks.shape[0]
    packed_q = blocks[:, 16:144]
    d = _f16_to_f32(blocks[:, 0:2])
    dmin = _f16_to_f32(blocks[:, 2:4])
    scale6, min6 = _unpack_scale_min_k4(blocks[:, 4:16])
    out = np.empty((num_blocks, QK_K), dtype=np.float32)
    for group in range(4):
        d1 = d * scale6[:, 2 * group]
        m1 = dmin * min6[:, 2 * group]
        d2 = d * scale6[:, 2 * group + 1]
        m2 = dmin * min6[:, 2 * group + 1]
        col = 32 * group + np.arange(32)
        lo = 64 * group
        out[:, lo:lo + 32] = d1[:, None] * (packed_q[:, col] & 0xF).astype(np.float32) - m1[:, None]
        out[:, lo + 32:lo + 64] = d2[:, None] * (packed_q[:, col] >> 4).astype(np.float32) - m2[:, None]
    return out


def dequantize_q5_K(blocks):
    # Q5_K (ggml-common.h block_q5_K, 176 bytes/256 elems):
    #   d/dmin (fp16), scales[12] (6-bit pairs), qh[32] high bits, qs[128] 4-bit.
    #   Same 8x32 structure as Q4_K plus a per-plane high bit that adds 16.
    num_blocks = blocks.shape[0]
    ql = blocks[:, 48:176]
    qh = blocks[:, 16:48]
    d = _f16_to_f32(blocks[:, 0:2])
    dmin = _f16_to_f32(blocks[:, 2:4])
    scale6, min6 = _unpack_scale_min_k4(blocks[:, 4:16])
    out = np.empty((num_blocks, QK_K), dtype=np.float32)
    for group in range(4):
        d1 = d * scale6[:, 2 * group]
        m1 = dmin * min6[:, 2 * group]
        d2 = d * scale6[:, 2 * group + 1]
        m2 = dmin * min6[:, 2 * group + 1]
        col = 32 * group + np.arange(32)
        hi_low = ((qh >> (2 * group)) & 1).astype(np.float32)
        hi_high = ((qh >> (2 * group + 1)) & 1).astype(np.float32)
        low = (ql[:, col] & 0xF).astype(np.float32)
        high = (ql[:, col] >> 4).astype(np.float32)
        lo = 64 * group
        out[:, lo:lo + 32] = d1[:, None] * (low + 16.0 * hi_low) - m1[:, None]
        out[:, lo + 32:lo + 64] = d2[:, None] * (high + 16.0 * hi_high) - m2[:, None]
    return out


def dequantize_q8_0(blocks):
    # Q8_0 (ggml-common.h block_q8_0, 34 bytes/32 elems):
    #   fp16 scale, then 32 signed int8 values; every value is q * d.
    d = _f16_to_f32(blocks[:, 0:2])
    packed_q = blocks[:, 2:34].view(np.int8)
    return d[:, None] * packed_q.astype(np.float32)


def dequantize_q2_K(blocks):
    # Q2_K (ggml-common.h block_q2_K, 84 bytes/256 elems):
    #   scales[16] (per-16 low nibble=scale, high nibble=min, 4-bit),
    #   qs[64] (2-bit packed quants), then fp16 block scale (d) and min (dmin).
    num_blocks = blocks.shape[0]
    scales = blocks[:, 0:16].astype(np.int32)
    packed_q = blocks[:, 16:80]
    block_scale = _f16_to_f32(blocks[:, 80:82])
    block_min = _f16_to_f32(blocks[:, 82:84])
    out = np.empty((num_blocks, QK_K), dtype=np.float32)
    is_ = 0
    for half in range(2):
        base = half * 32
        for j in range(4):
            shift = j * 2
            for sub in range(2):
                sc = scales[:, is_]
                is_ += 1
                group_scale = block_scale * (sc & 0xF)
                group_min = block_min * (sc >> 4)
                cols = base + sub * 16 + np.arange(16)
                qv = ((packed_q[:, cols] >> shift) & 3).astype(np.float32)
                lo = half * 128 + j * 32 + sub * 16
                out[:, lo:lo + 16] = group_scale[:, None] * qv - group_min[:, None]
    return out


def dequantize_q3_K(blocks):
    # Q3_K (ggml-common.h block_q3_K, 110 bytes/256 elems):
    #   hmask[32] high bits, qs[64] low 2 bits, scales[12] (sixteen 6-bit scales
    #   packed into 12 bytes, reassembled by the kmask shuffle below), fp16 scale.
    num_blocks = blocks.shape[0]
    hmask = blocks[:, 0:32]
    packed_q = blocks[:, 32:96]
    scale_bytes = blocks[:, 96:108]
    block_scale = _f16_to_f32(blocks[:, 108:110])

    # Sixteen 6-bit scales live packed in 12 bytes: three 32-bit words, then a
    # bit-shuffle reassembles them (kmask1/kmask2 from ggml-quants.c).
    a0 = (scale_bytes[:, 0].astype(np.uint32) | (scale_bytes[:, 1].astype(np.uint32) << 8)
          | (scale_bytes[:, 2].astype(np.uint32) << 16) | (scale_bytes[:, 3].astype(np.uint32) << 24))
    a1 = (scale_bytes[:, 4].astype(np.uint32) | (scale_bytes[:, 5].astype(np.uint32) << 8)
          | (scale_bytes[:, 6].astype(np.uint32) << 16) | (scale_bytes[:, 7].astype(np.uint32) << 24))
    a2 = (scale_bytes[:, 8].astype(np.uint32) | (scale_bytes[:, 9].astype(np.uint32) << 8)
          | (scale_bytes[:, 10].astype(np.uint32) << 16) | (scale_bytes[:, 11].astype(np.uint32) << 24))
    tmp = a2
    a2n = ((a0 >> 4) & _MASK_KMASK2) | (((tmp >> 4) & _MASK_KMASK1) << 4)
    a3n = ((a1 >> 4) & _MASK_KMASK2) | (((tmp >> 6) & _MASK_KMASK1) << 4)
    a0n = (a0 & _MASK_KMASK2) | (((tmp >> 0) & _MASK_KMASK1) << 4)
    a1n = (a1 & _MASK_KMASK2) | (((tmp >> 2) & _MASK_KMASK1) << 4)

    row_scales = np.empty((num_blocks, 16), dtype=np.int8)
    for i, word in enumerate((a0n, a1n, a2n, a3n)):
        row_scales[:, i * 4:(i + 1) * 4] = word.astype("<u4").view(np.uint8).reshape(num_blocks, 4)

    out = np.empty((num_blocks, QK_K), dtype=np.float32)
    is_ = 0
    for half in range(2):
        base = half * 32
        for j in range(4):
            shift = j * 2
            m = 1 << (half * 4 + j)
            for sub in range(2):
                group_scale = block_scale * (row_scales[:, is_] - 32).astype(np.float32)
                is_ += 1
                qcols = base + sub * 16 + np.arange(16)
                hcols = sub * 16 + np.arange(16)
                qv = ((packed_q[:, qcols] >> shift) & 3).astype(np.float32)
                high_bit = ((hmask[:, hcols] & m) != 0).astype(np.float32)
                # low 2-bit code, minus 4 when the high bit of the mask is clear
                centered_q = qv - 4.0 * (1.0 - high_bit)
                lo = half * 128 + j * 32 + sub * 16
                out[:, lo:lo + 16] = group_scale[:, None] * centered_q
    return out


def dequantize_q6_K(blocks):
    # Q6_K (ggml-common.h block_q6_K, 210 bytes/256 elems):
    #   ql[128] low 4 bits, qh[64] upper 2 bits, scales[16] int8 per-16 scales,
    #   fp16 block scale. Two 128-elem halves; codes are (low | high<<4) - 32.
    num_blocks = blocks.shape[0]
    packed_lo = blocks[:, 0:128]
    packed_hi = blocks[:, 128:192]
    row_scales = blocks[:, 192:208].view(np.int8)
    block_scale = _f16_to_f32(blocks[:, 208:210])
    out = np.empty((num_blocks, QK_K), dtype=np.float32)
    for half in range(2):
        lo_base = half * 64
        hi_base = half * 32
        scale_base = half * 8
        l = np.arange(32)
        group = l // 16
        code0 = ((packed_lo[:, lo_base + l] & 0xF).astype(np.int32)
                 | (((packed_hi[:, hi_base + l] >> 0) & 3).astype(np.int32) << 4)) - 32
        code1 = ((packed_lo[:, lo_base + l + 32] & 0xF).astype(np.int32)
                 | (((packed_hi[:, hi_base + l] >> 2) & 3).astype(np.int32) << 4)) - 32
        code2 = ((packed_lo[:, lo_base + l] >> 4).astype(np.int32)
                 | (((packed_hi[:, hi_base + l] >> 4) & 3).astype(np.int32) << 4)) - 32
        code3 = ((packed_lo[:, lo_base + l + 32] >> 4).astype(np.int32)
                 | (((packed_hi[:, hi_base + l] >> 6) & 3).astype(np.int32) << 4)) - 32
        scale0 = row_scales[:, scale_base + group + 0].astype(np.float32)
        scale1 = row_scales[:, scale_base + group + 2].astype(np.float32)
        scale2 = row_scales[:, scale_base + group + 4].astype(np.float32)
        scale3 = row_scales[:, scale_base + group + 6].astype(np.float32)
        base = half * 128
        out[:, base + l] = block_scale[:, None] * scale0 * code0.astype(np.float32)
        out[:, base + l + 32] = block_scale[:, None] * scale1 * code1.astype(np.float32)
        out[:, base + l + 64] = block_scale[:, None] * scale2 * code2.astype(np.float32)
        out[:, base + l + 96] = block_scale[:, None] * scale3 * code3.astype(np.float32)
    return out


_DEQUANTIZERS = {
    GGML_TYPE_Q2_K: dequantize_q2_K,
    GGML_TYPE_Q3_K: dequantize_q3_K,
    GGML_TYPE_Q4_K: dequantize_q4_K,
    GGML_TYPE_Q5_K: dequantize_q5_K,
    GGML_TYPE_Q6_K: dequantize_q6_K,
    GGML_TYPE_Q8_0: dequantize_q8_0,
}


def dequantize(raw, ggml_type, numel):
    """Decode a tensor's stored bytes to a flat float32 array (GGUF element order)."""
    if ggml_type in _DEQUANTIZERS:
        block_elems, block_bytes = ggml_block_spec(ggml_type)
        if numel % block_elems != 0:
            raise GGUFError(
                "dequant %s: numel %d is not a multiple of block %d"
                % (ggml_type_name(ggml_type), numel, block_elems))
        nblocks = numel // block_elems
        data = np.frombuffer(raw, dtype=np.uint8)
        if data.size != nblocks * block_bytes:
            raise GGUFError(
                "dequant %s: %d bytes for %d blocks of %d, expected %d"
                % (ggml_type_name(ggml_type), data.size, nblocks, block_bytes,
                   nblocks * block_bytes))
        blocks = data.reshape(nblocks, block_bytes)
        return _DEQUANTIZERS[ggml_type](blocks).reshape(-1)

    element_size = {GGML_TYPE_F32: 4, GGML_TYPE_F16: 2, GGML_TYPE_BF16: 2}.get(ggml_type)
    if element_size is not None:
        expected = numel * element_size
        if len(raw) != expected:
            raise GGUFError(
                "dequant %s: %d bytes for %d elements, expected %d"
                % (ggml_type_name(ggml_type), len(raw), numel, expected))
        if ggml_type == GGML_TYPE_F32:
            return np.frombuffer(raw, dtype="<f4").astype(np.float32)
        if ggml_type == GGML_TYPE_F16:
            return np.frombuffer(raw, dtype="<f2").astype(np.float32)
        return _bf16_to_f32(np.frombuffer(raw, dtype=np.uint8))

    raise GGUFError("dequant: unsupported ggml type %d (%s)"
                    % (ggml_type, ggml_type_name(ggml_type)))