Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
7.2 KB
Raw Blame
// src/kernels/cpu/q8k_quant_parity.cpp - the AVX-2 Q8_K activation quantizer (q8k_quant_avx2, iq_avx2.cpp) against
// ggml's, byte for byte: the reference (ggml-base's from_float_ref) and what ggml-cpu runs (its from_float), over
// rows of random, scaled, sparse and degenerate values; then the same through native_quant_act / native_quant_h with
// an IQ3_S expert's format, which is what the engine calls.  No GPU, no model.
//
//     q8k_quant_parity            (STRATA_FORCE_ISA=avx: the AVX-2 copy is skipped, native_quant_* must equal ggml's)
//     q8k_quant_parity --bench    ns per 2560-value row, ggml-cpu's and the AVX-2 copy
#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/cpu/iq_avx2.hpp"
#include "strata/kernels/cpu/iq_avx1.hpp"
#include "strata/kernels/cpu/native_expert.hpp"

#include "ggml.h"
#include "ggml-cpu.h"

#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <limits>
#include <random>
#include <string>
#include <vector>

namespace cpu = strata::kernels::cpu;

// ggml-quants.h (ggml-base): the scalar reference, which ggml-cpu's x86 quantize_row_q8_K calls.  Q8_K has no
// from_float_ref in ggml's type traits, so it is declared here (block_q8_K* in ggml; the bytes are all this reads).
extern "C" void quantize_row_q8_K_ref(const float* x, void* y, int64_t k);

namespace {

constexpr int kQK = 256;

std::vector<std::vector<float>> cases(int n) {
    std::mt19937 rng(11);
    std::normal_distribution<float> nd(0.f, 1.f);
    std::uniform_real_distribution<float> ud(-100.f, 100.f);
    std::vector<std::vector<float>> c;
    auto add = [&](auto fill) {
        std::vector<float> x((size_t) n);
        for (int j = 0; j < n; ++j) x[(size_t) j] = fill(j);
        c.push_back(std::move(x));
    };
    add([&](int) { return nd(rng); });
    add([&](int) { return nd(rng) * 1e-3f; });
    add([&](int) { return nd(rng) * 1e4f; });
    add([&](int) { return ud(rng); });
    add([&](int) { return 0.f; });                                         // every block zero
    add([&](int j) { return (j / kQK) % 2 ? nd(rng) : 0.f; });             // zero blocks between random ones
    add([&](int j) { return j % kQK == 77 ? nd(rng) : 0.f; });             // one value per block
    add([&](int j) {                                                       // the largest magnitude twice, the
        const int k = j % kQK;                                             // FIRST one's sign wins
        return k == 3 ? -5.f : k == 100 ? 5.f : nd(rng);
    });
    add([&](int j) {
        const int k = j % kQK;
        return k == 3 ? 5.f : k == 100 ? -5.f : nd(rng);
    });
    add([&](int j) { return j % kQK == 9 ? 3e38f : nd(rng); });            // extremes beside ordinary values
    add([&](int j) { return j % kQK == 9 ? -3e38f : nd(rng) * 1e30f; });
    add([&](int) { return nd(rng) * 1e-40f; });                            // denormals
    add([&](int j) { return (float) ((j % 255) - 127) / 127.f; });         // the grid itself, and its ties
    add([&](int j) { return ((float) ((j * 37) % 509) - 254.f) / 508.f; });
    add([&](int j) { return j % kQK == 5 ? std::numeric_limits<float>::quiet_NaN() : nd(rng); });
    add([&](int j) { return j % kQK == 5 ? std::numeric_limits<float>::infinity() : nd(rng); });
    return c;
}

// both outputs prefilled alike: a zero block leaves bsums unwritten in the reference, and so must the copy
bool same(const char* tag, int n, int ci, ggml_from_float_t a, void (*b)(const float*, void*, int64_t),
          const std::vector<float>& x, size_t bytes) {
    std::vector<uint8_t> ya(bytes, 0xAB), yb(bytes, 0xAB);
    a(x.data(), ya.data(), n);
    b(x.data(), yb.data(), n);
    if (std::memcmp(ya.data(), yb.data(), bytes) == 0) return true;
    size_t at = 0;
    while (ya[at] == yb[at]) ++at;
    std::printf("  %s n=%d case %d: MISMATCH at byte %zu of %zu\n", tag, n, ci, at, bytes);
    return false;
}

const cpu::NativeFmt* g_fmt = nullptr;
void via_act(const float* x, void* y, int64_t) { cpu::native_quant_act(*g_fmt, x, y); }
void via_h(const float* x, void* y, int64_t) { cpu::native_quant_h(*g_fmt, x, y); }

}  // namespace

int main(int argc, char** argv) {
    const bool bench = argc > 1 && std::string(argv[1]) == "--bench";
    ggml_cpu_init();
    const ggml_from_float_t ref = quantize_row_q8_K_ref;
    const ggml_from_float_t gcpu = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K)->from_float;
    const bool avx2 = cpu::cpu_avx2_ok();
    const bool avx1 = !avx2 && cpu::cpu_avx1_ok();   // the older CPUs' copy (checked wherever AVX exists)
    int failures = 0, checked = 0;
    for (const int n : {256, 512, 2560, 6144}) {
        const size_t bytes = ggml_row_size(GGML_TYPE_Q8_K, n);
        const auto cs = cases(n);
        for (int ci = 0; ci < (int) cs.size(); ++ci) {
            failures += !same("ggml-cpu vs reference", n, ci, ref, gcpu, cs[(size_t) ci], bytes);
            if (avx2) failures += !same("AVX-2 vs reference", n, ci, ref, cpu::q8k_quant_avx2, cs[(size_t) ci], bytes);
            if (avx1) failures += !same("AVX1 vs reference", n, ci, ref, cpu::q8k_quant_avx1, cs[(size_t) ci], bytes);
            ++checked;
        }
    }
    // the engine's entry points: an IQ3_S expert (type 21) quantizes both of its activations to Q8_K
    cpu::NativeFmt f;
    std::string err;
    if (!cpu::native_fmt(21, 21, 2560, 512, f, err)) {
        std::printf("native_fmt: %s\n", err.c_str());
        return 1;
    }
    g_fmt = &f;
    for (const auto& x : cases(2560))
        failures += !same("native_quant_act", 2560, 0, ref, via_act, x, ggml_row_size(GGML_TYPE_Q8_K, 2560));
    for (const auto& x : cases(512))
        failures += !same("native_quant_h", 512, 0, ref, via_h, x, ggml_row_size(GGML_TYPE_Q8_K, 512));
    std::printf("q8k_quant_parity: %d rows x 4 sizes, AVX-2 copy %s, AVX1 copy %s: %s\n", checked / 4,
                avx2 ? "checked" : "skipped (no AVX2, or STRATA_FORCE_ISA)",
                avx1 ? "checked" : "skipped (AVX2 present, no AVX, or STRATA_FORCE_ISA)", failures ? "FAILED" : "identical");
    if (bench && (avx2 || avx1)) {
        std::mt19937 rng(3);
        std::normal_distribution<float> nd(0.f, 1.f);
        const int n = 2560, rows = 4096;
        std::vector<float> x((size_t) n * rows);
        for (auto& v : x) v = nd(rng);
        const size_t bytes = ggml_row_size(GGML_TYPE_Q8_K, n);
        std::vector<uint8_t> y(bytes * rows);
        auto time = [&](const char* tag, auto fn) {
            double best = 1e30;
            for (int rep = 0; rep < 5; ++rep) {
                const auto t0 = std::chrono::steady_clock::now();
                for (int r = 0; r < rows; ++r) fn(x.data() + (size_t) r * n, y.data() + bytes * r, n);
                const double ns = std::chrono::duration<double, std::nano>(std::chrono::steady_clock::now() - t0).count();
                best = std::min(best, ns / rows);
            }
            std::printf("  %-10s %7.0f ns per %d-value row\n", tag, best, n);
            return best;
        };
        const double a = time("ggml-cpu", gcpu);
        const double b = avx2 ? time("AVX-2", cpu::q8k_quant_avx2) : time("AVX1", cpu::q8k_quant_avx1);
        std::printf("  %.1fx\n", a / b);
    }
    return failures ? 1 : 0;
}