Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fix: GGUF Q5_1 quant crashes llamafile on aarch64 cpu (#928)
Davide Eynarde08323fc6702
diff --git a/llamafile/iqk_mul_mat.inc b/llamafile/iqk_mul_mat.incindex 47c92e51..8043cec6 100644--- a/llamafile/iqk_mul_mat.inc+++ b/llamafile/iqk_mul_mat.inc@@ -2652,13 +2652,14 @@ template <int nrc> struct Q81 {return y[iy][4*i].qs;}- // Gather d,s pairs from 4 individual blocks into a float16x8_t- // Layout: [d0, s0, d1, s1, d2, s2, d3, s3]+ // Gather d and s values from 4 individual blocks into a float16x8_t+ // Layout: [d0, d1, d2, d3, s0, s1, s2, s3]+ // Must match the qx_scales layout from DequantizerQ*1::new_block()inline float16x8_t load_scales(int iy, int i) const {ggml_half ds_vals[8];for (int k = 0; k < 4; ++k) {- ds_vals[2*k + 0] = y[iy][4*i + k].d;- ds_vals[2*k + 1] = y[iy][4*i + k].s;+ ds_vals[k] = y[iy][4*i + k].d;+ ds_vals[4 + k] = y[iy][4*i + k].s;}return vld1q_f16((const float16_t *)ds_vals);}@@ -2840,7 +2841,7 @@ struct DequantizerQ51 final : public BaseLegacyDequantizer<block_q5_1> {q[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[1]), vandq_u8(mh, hbits.to_bytes(qh+2))));}inline void prepare1(int i) {- bits.prepare1(x[i].qs, bits.b);+ prepare1(i, bits.b);}inline float16x8_t new_block(int i) {diff --git a/tests/sgemm/BUILD.mk b/tests/sgemm/BUILD.mkindex 801cb792..ba6388bf 100644--- a/tests/sgemm/BUILD.mk+++ b/tests/sgemm/BUILD.mk@@ -78,6 +78,21 @@ o/$(MODE)/tests/sgemm/iqk_test: \@mkdir -p $(@D)$(CXX) $(LDFLAGS) -fopenmp -o $@ $^ $(LDLIBS)+# ==============================================================================+# Test: iqk_legacy_test (legacy quant types Q4_0/Q4_1/Q5_0/Q5_1)+# ==============================================================================+# Reproduces https://github.com/mozilla-ai/llamafile/issues/922++o/$(MODE)/tests/sgemm/iqk_legacy_test.o: tests/sgemm/iqk_legacy_test.cpp+ @mkdir -p $(@D)+ $(CXX) $(CXXFLAGS) $(SGEMM_TEST_CPPFLAGS) -c -o $@ $<++o/$(MODE)/tests/sgemm/iqk_legacy_test: \+ o/$(MODE)/tests/sgemm/iqk_legacy_test.o \+ $(SGEMM_TEST_DEPS)+ @mkdir -p $(@D)+ $(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)+# ==============================================================================# Test: q8_0_layout_test (standalone diagnostic)# ==============================================================================@@ -104,4 +119,5 @@ o/$(MODE)/tests/sgemm: \o/$(MODE)/tests/sgemm/sgemm_sss_test \o/$(MODE)/tests/sgemm/sgemm_matmul_test \o/$(MODE)/tests/sgemm/sgemm_vecdot_test \- o/$(MODE)/tests/sgemm/iqk_test+ o/$(MODE)/tests/sgemm/iqk_test \+ o/$(MODE)/tests/sgemm/iqk_legacy_testdiff --git a/tests/sgemm/iqk_legacy_test.cpp b/tests/sgemm/iqk_legacy_test.cppnew file mode 100644index 00000000..97c3df5d--- /dev/null+++ b/tests/sgemm/iqk_legacy_test.cpp@@ -0,0 +1,241 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Test for legacy quant types (Q4_0, Q4_1, Q5_0, Q5_1) through IQK path.+// Reproduces https://github.com/mozilla-ai/llamafile/issues/922+//+// Tests two classes of bugs:+// 1. Q81::load_scales layout mismatch (affects Q4_1, Q5_1 at all dimensions)+// 2. DequantizerQ51::prepare1(int) missing 5th bit (affects Q5_1 when nb%4 != 0)++#include "sgemm.h"+#include "ggml.h"+#include "ggml-quants.h"++#define GGML_COMMON_DECL_C+#include "ggml-common.h"++#define GGML_COMMON_IMPL_C+#include "ggml-common.h"++#include <cassert>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <thread>++static const long Nx = 32;+static const long Ny = 64;++static unsigned long long lcg = 12345;++static inline float rand_float(void) {+ lcg *= 6364136223846793005;+ lcg += 1442695040888963407;+ return (float)((int)(lcg >> 32) % 10000 - 5000) / 10000.0f;+}++// Dequantize Q8_1 (not provided by ggml-quants.h)+static void dequantize_row_q8_1(const block_q8_1 *x, float *y, int64_t k) {+ assert(k % QK8_1 == 0);+ const int nb = k / QK8_1;+ for (int i = 0; i < nb; i++) {+ const float d = ggml_fp16_to_fp32(x[i].d);+ for (int j = 0; j < QK8_1; ++j) {+ y[i * QK8_1 + j] = x[i].qs[j] * d;+ }+ }+}++static bool iqk_mul_mat_wrapper(long Nx, long Ny, long ne00, int typeA,+ const void *A, const void *B, float *C, long stride_C) {+ int nth = std::thread::hardware_concurrency();+ bool all_ok = true;+ for (int ith = 0; ith < nth; ++ith) {+#if defined(__x86_64__)+ bool res = iqk_mul_mat(Nx, Ny, ne00, typeA, A, B, C, stride_C, ith, nth);+#elif defined(__aarch64__)+ bool res = iqk_mul_mat_arm82(Nx, Ny, ne00, typeA, A, B, C, stride_C, ith, nth);+#else+ bool res = false;+#endif+ all_ok = all_ok && res;+ }+ return all_ok;+}++struct QuantType {+ int type;+ const char *name;+ void (*quantize)(const float *, void *, int64_t);+ void (*dequantize)(const void *, float *, int64_t);+ int q8_type;+};++static void quantize_q4_0(const float *x, void *y, int64_t k) {+ quantize_row_q4_0_ref(x, (block_q4_0 *)y, k);+}+static void dequantize_q4_0(const void *x, float *y, int64_t k) {+ dequantize_row_q4_0((const block_q4_0 *)x, y, k);+}+static void quantize_q4_1(const float *x, void *y, int64_t k) {+ quantize_row_q4_1_ref(x, (block_q4_1 *)y, k);+}+static void dequantize_q4_1(const void *x, float *y, int64_t k) {+ dequantize_row_q4_1((const block_q4_1 *)x, y, k);+}+static void quantize_q5_0(const float *x, void *y, int64_t k) {+ quantize_row_q5_0_ref(x, (block_q5_0 *)y, k);+}+static void dequantize_q5_0(const void *x, float *y, int64_t k) {+ dequantize_row_q5_0((const block_q5_0 *)x, y, k);+}+static void quantize_q5_1(const float *x, void *y, int64_t k) {+ quantize_row_q5_1_ref(x, (block_q5_1 *)y, k);+}+static void dequantize_q5_1(const void *x, float *y, int64_t k) {+ dequantize_row_q5_1((const block_q5_1 *)x, y, k);+}++static QuantType quant_types[] = {+ {GGML_TYPE_Q4_0, "Q4_0", quantize_q4_0, dequantize_q4_0, GGML_TYPE_Q8_0},+ {GGML_TYPE_Q4_1, "Q4_1", quantize_q4_1, dequantize_q4_1, GGML_TYPE_Q8_1},+ {GGML_TYPE_Q5_0, "Q5_0", quantize_q5_0, dequantize_q5_0, GGML_TYPE_Q8_0},+ {GGML_TYPE_Q5_1, "Q5_1", quantize_q5_1, dequantize_q5_1, GGML_TYPE_Q8_1},+};++// Test a specific quant type with a given inner dimension.+// ne00 must be a multiple of 32.+static int test_type(const QuantType &qt, long ne00) {+ int nb = ne00 / 32;+ printf("\n--- %s ne00=%ld (nb=%d, nb%%4=%d) ---\n", qt.name, ne00, nb, nb % 4);++ float *src_A = (float *)aligned_alloc(64, Nx * ne00 * sizeof(float));+ float *src_B = (float *)aligned_alloc(64, Ny * ne00 * sizeof(float));++ lcg = 12345;+ for (long i = 0; i < Nx * ne00; ++i) src_A[i] = rand_float();+ for (long i = 0; i < Ny * ne00; ++i) src_B[i] = rand_float();++ size_t row_size_A = ggml_row_size((ggml_type)qt.type, ne00);+ void *A_quant = aligned_alloc(64, Nx * row_size_A);+ for (long i = 0; i < Nx; ++i) {+ qt.quantize(src_A + i * ne00, (char *)A_quant + i * row_size_A, ne00);+ }++ size_t row_size_B = ggml_row_size((ggml_type)qt.q8_type, ne00);+ void *B_quant = aligned_alloc(64, Ny * row_size_B);+ for (long j = 0; j < Ny; ++j) {+ if (qt.q8_type == GGML_TYPE_Q8_0) {+ quantize_row_q8_0_ref(src_B + j * ne00, (block_q8_0 *)((char *)B_quant + j * row_size_B), ne00);+ } else {+ quantize_row_q8_1_ref(src_B + j * ne00, (block_q8_1 *)((char *)B_quant + j * row_size_B), ne00);+ }+ }++ long stride_C = Nx;+ float *C_iqk = (float *)aligned_alloc(64, Ny * stride_C * sizeof(float));+ memset(C_iqk, 0, Ny * stride_C * sizeof(float));++ bool ok = iqk_mul_mat_wrapper(Nx, Ny, ne00, qt.type, A_quant, B_quant, C_iqk, stride_C);+ if (!ok) {+ printf(" IQK not supported, skipping\n");+ free(C_iqk); free(B_quant); free(A_quant); free(src_B); free(src_A);+ return 0;+ }++ // Reference: dequantize both sides and do float matmul+ float *A_float = (float *)aligned_alloc(64, Nx * ne00 * sizeof(float));+ float *B_float = (float *)aligned_alloc(64, Ny * ne00 * sizeof(float));++ for (long i = 0; i < Nx; ++i) {+ qt.dequantize((char *)A_quant + i * row_size_A, A_float + i * ne00, ne00);+ }+ for (long j = 0; j < Ny; ++j) {+ if (qt.q8_type == GGML_TYPE_Q8_0) {+ dequantize_row_q8_0((const block_q8_0 *)((char *)B_quant + j * row_size_B),+ B_float + j * ne00, ne00);+ } else {+ dequantize_row_q8_1((const block_q8_1 *)((char *)B_quant + j * row_size_B),+ B_float + j * ne00, ne00);+ }+ }++ float *C_ref = (float *)aligned_alloc(64, Ny * stride_C * sizeof(float));+ for (long j = 0; j < Ny; ++j) {+ for (long i = 0; i < Nx; ++i) {+ double sum = 0.0;+ for (long k = 0; k < ne00; ++k) {+ sum += (double)A_float[i * ne00 + k] * (double)B_float[j * ne00 + k];+ }+ C_ref[j * stride_C + i] = (float)sum;+ }+ }++ int nan_count = 0;+ int inf_count = 0;+ double err_sum = 0.0;+ double err_max = 0.0;+ long err_count = 0;++ for (long j = 0; j < Ny; ++j) {+ for (long i = 0; i < Nx; ++i) {+ float iqk_val = C_iqk[j * stride_C + i];+ float ref_val = C_ref[j * stride_C + i];++ if (std::isnan(iqk_val)) { nan_count++; continue; }+ if (std::isinf(iqk_val)) { inf_count++; continue; }++ double rel_err = std::fabs(iqk_val - ref_val) / (std::fabs(ref_val) + 1e-6);+ err_sum += rel_err;+ if (rel_err > err_max) err_max = rel_err;+ ++err_count;+ }+ }++ int rc = 0;+ if (nan_count > 0 || inf_count > 0) {+ printf(" FAIL: %d NaN, %d Inf\n", nan_count, inf_count);+ rc = 1;+ } else if (err_max > 0.05) {+ printf(" FAIL: max error %.2e exceeds threshold\n", err_max);+ rc = 1;+ } else {+ printf(" PASS (avg=%.2e max=%.2e)\n", err_sum / err_count, err_max);+ }++ free(C_ref); free(B_float); free(A_float);+ free(C_iqk); free(B_quant); free(A_quant);+ free(src_B); free(src_A);+ return rc;+}++int main() {+ printf("iqk_legacy_test: Testing legacy quant types through IQK path\n");+ printf("Threads: %u\n", std::thread::hardware_concurrency());++ // ne00 must be a multiple of 256 because iqk_mul_mat unconditionally+ // computes ggml_row_size(Q8_K, ne00) which asserts ne00 % 256 == 0.+ // Since 256/32 = 8 blocks and 8%4 == 0, the process_1_block remainder+ // path (nb%4 != 0) is unreachable for legacy quants through this API.+ // The prepare1 fix for DequantizerQ51 is still correct but cannot be+ // exercised here. We test multiple dimensions for general coverage.+ static const long test_dims[] = { 256, 512, 1024 };++ int failures = 0;+ for (const auto &qt : quant_types) {+ for (long ne00 : test_dims) {+ if (test_type(qt, ne00) != 0)+ failures++;+ }+ }++ printf("\n============================================\n");+ if (failures > 0) {+ printf("FAILED: %d test(s) failed\n", failures);+ return 1;+ }+ printf("All tests passed.\n");+ return 0;+}