Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/iq_nibble_parity.cpp - STRATA_EXPERT_NIBBLE (strata/core/expert_nibble.hpp): the 4-bit gate/up expert
// slots against the production kernels and the GGUF blobs, bit for bit.
//
// build/iq_nibble_parity --header the host half of expert_nibble.hpp (no GPU)
// build/iq_nibble_parity <shard.gguf> [<shard.gguf> ...] [--layers-per-fmt 2] [--layers 1,17] [--ne 512]
// [--plans 8] [--synthetic 64] [--no-ggml] [--negative]
// build/iq_nibble_parity --synthetic-only [--synthetic 64] [--plans 8] [--no-ggml] no model, no shard
//
// Each model (any shard of it; its other shards are opened beside it) gives `--layers-per-fmt` layers of each
// converted format (IQ2_S, IQ3_XXS, IQ3_S; or `--layers`), `--ne` experts each, read from the GGUF tensors into the
// blob the pack holds (gate rows, up rows, down rows). Per layer:
// (1) nibble::transcode, 64 blobs a batch as the startup fill runs it: 0 items that do not round-trip;
// (2) every gate/up byte of a slot is written (8 slots transcoded again into another pre-fill byte);
// (3) nibble::hash_slots against nibble::hash_host on every slot read back (and a flipped byte must change it);
// (4) the host codec on every item: decode_item then encode_item gives the code back;
// (5) ggml's reference dequantizer (to_float) on every GGUF row against the weight the 4-bit slot gives through
// dequantize_row_iq2_s / _iq3_xxs / _iq3_s's own expression (code, scale byte, fp16 d): every weight, bit for bit;
// (6) nibble::check_gguf (the start-up check: the GGUF blocks rebuilt from every slot against the blob): 0 words
// differ; nibble::gather_gguf (the prompt path's MMQ gather) in groups of 16 and of 1..15 from a later first
// position: the rebuilt [gate | up] and the down rows byte for byte the blob's, the MMQ tail after the group zero;
// (7) native_expert_grouped on the converted slots (native_expert_layout_nib) against the same call on the GGUF
// blobs (native_expert_layout): hq and out byte for byte for every entry, and gate / up where the reference
// layout writes them - in the card's default layout (a GP100: mode 8, whose gate/up kernel writes hq) and in the
// CUDA layout (native_expert_set_mode(0)), each with native_grouped_set_v1 off and on (h too, in mode 0 + v1);
// top-10 routing T = 1..6 at 0% and 50% sharing (`--plans` each), one group of T entries (G = 1) per T, plans of
// groups with 0..11 entries (more than one GRP_NC pass, empty groups); grid_groups 0 (cap) and 4 (the PCIe
// call's stride); the default kernel per format and each skeleton forced (S, P); q8_1 activations random, all
// +127, all -127 and alternating +-127 (the sum of x at its extremes, +-4064).
// `--synthetic N`: per format, N experts whose gate/up blocks are random bytes (every grid index, sign byte and scale
// nibble the format has; the coverage is counted and must be complete) and whose fp16 d cycle through +-the smallest
// subnormal, +-65504, +-1, the smallest normal and two ordinary values; checks (1)-(7) on them.
// `--synthetic-only`: the synthetic experts alone, for every gate/up format x both down types (IQ4_NL, Q2_0), the down rows
// random bytes with ordinary fp16 scales: checks (1)-(7) with no model file (a GP100 is still needed).
// `--negative`: the control - one code byte per slot changed after the transcode; (6) and (7) must then report differences.
// Exit 0 when every check is equal; 1 otherwise; 2 on a usage or setup error. A build without STRATA_EXPERT_NIBBLE_BUILT
// (the CUDA build with STRATA_EXPERIMENTAL_SM60) has no 4-bit kernels: --header only.
#include "strata/artifact/gguf_reader.hpp"
#include "strata/core/expert_nibble.hpp"
#include "strata/kernels/cpu/native_expert.hpp"
#include "strata/kernels/iq_kernels.hpp"
#include "ggml.h"
#include <cuda_runtime.h>
#include <algorithm>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <thread>
#include <vector>
namespace {
namespace nib = strata::core::nibble;
namespace K = strata::kernels;
namespace cpu = strata::kernels::cpu;
int fails = 0;
void expect(bool ok, const char* what) {
if (!ok) {
std::printf("FAIL %s\n", what);
++fails;
}
}
// Every value the format holds at every one of an item's 32 positions (the other positions cycle through the set),
// encoded, decoded and compared; and one value outside the set must be refused.
void round_trip(int fmt, const std::vector<int8_t>& set, int8_t outside) {
int8_t g[32], back[32];
uint32_t q[4];
for (size_t shift = 0; shift < set.size(); ++shift) {
for (int i = 0; i < 32; ++i) g[i] = set[(shift + (size_t) i) % set.size()];
const bool ok = nib::encode_item(fmt, g, q);
nib::decode_item(fmt, q, back);
expect(ok && std::memcmp(g, back, 32) == 0, (std::string(nib::fmt_name(fmt)) + " round trip").c_str());
}
for (int i = 0; i < 32; ++i) g[i] = set[0];
g[17] = outside;
expect(!nib::encode_item(fmt, g, q), (std::string(nib::fmt_name(fmt)) + " refuses a value outside the set").c_str());
}
int header_check() {
expect(nib::kGuBytes == 1766400 && nib::kScOff == 1638400 && nib::kDOff == 1740800, "layout sizes");
expect(nib::code_off(nib::kNFf, 0) == nib::kUpCodesOff, "up rows' codes");
expect(nib::slot_bytes_for(921600) % nib::kSlotAlign == 0, "slot alignment");
round_trip(nib::kIQ2_S, { 8, 25, 43, -8, -25, -43 }, 9);
round_trip(nib::kIQ3_XXS, { 4, 12, 20, 28, 36, 44, 52, 62, -4, -12, -20, -28, -36, -44, -52, -62 }, 5);
std::vector<int8_t> s3;
for (int v = -15; v <= 15; v += 2) s3.push_back((int8_t) v);
round_trip(nib::kIQ3_S, s3, 2);
// IQ2_S's nibble is the decode kernel's PRMT selector into {8,25,43,0} {-8,-25,-43,0}
{
int8_t g[32] = {}, back[32];
uint32_t q[4];
for (int i = 0; i < 32; ++i) g[i] = (int8_t) (i % 2 ? -25 : 43);
nib::encode_item(nib::kIQ2_S, g, q);
expect((q[0] & 0xFu) == 2u && (q[0] >> 4 & 0xFu) == 5u, "IQ2_S selector nibbles");
nib::decode_item(nib::kIQ2_S, q, back);
expect(std::memcmp(g, back, 32) == 0, "IQ2_S selector decode");
}
expect(nib::gu_skeleton(nib::kIQ2_S) == nib::kSkelP && nib::gu_skeleton(nib::kIQ3_XXS) == nib::kSkelS &&
nib::gu_skeleton(nib::kIQ3_S) == nib::kSkelS,
"kernel table");
{
const uint8_t b[6] = { 1, 2, 3, 4, 5, 6 };
const uint64_t h = nib::hash_host(b, 6);
const uint64_t want = (uint64_t) (0x04030201u ^ 0u) * nib::kHashMul + (uint64_t) (0x00000605u ^ 1u) * nib::kHashMul;
expect(h == want, "hash_host");
}
std::printf("%s (%d failures)\n", fails == 0 ? "header OK" : "header FAILED", fails);
return fails == 0 ? 0 : 1;
}
// ------------------------------------------------------------------------------------------------ the GPU parity
#define CK(x) \
do { \
cudaError_t e_ = (x); \
if (e_ != cudaSuccess) { \
std::fprintf(stderr, "%s:%d %s: %s\n", __FILE__, __LINE__, #x, cudaGetErrorString(e_)); \
std::exit(2); \
} \
} while (0)
struct Q81 { // block_q8_1: fp16 d, fp16 s (= d * sum), 32 int8
uint16_t d, s;
int8_t qs[32];
};
static_assert(sizeof(Q81) == 36, "block_q8_1");
constexpr int64_t kEmbd = nib::kNEmbd, kFf = nib::kNFf;
constexpr int kTok = 12; // activation rows: T <= 6 for routing plans, 12 for the 0..11-entry plans
// One layer: `ne` GGUF blobs (gate rows | up rows | down rows, as the pack and the host arena hold them) on the host
// and the device, and their converted slots.
struct Layer {
std::string tag;
int layer = -1;
cpu::NativeFmt f;
nib::LayerVram V;
K::NativeExpertLayout Lr, Ln;
int ne = 0;
std::vector<uint8_t> host; // ne x f.bytes
uint8_t* raw = nullptr; // device copy of `host`
uint8_t* slots = nullptr; // ne x V.slot_bytes (the last one only `hashed` long: nothing past it is readable)
};
bool layer_fmt(const strata::GgufModel& m, int l, int& gt, int& dt) {
const auto* g = m.find("blk." + std::to_string(l) + ".ffn_gate_exps.weight");
const auto* d = m.find("blk." + std::to_string(l) + ".ffn_down_exps.weight");
if (g == nullptr || d == nullptr) return false;
gt = (int) g->type;
dt = (int) d->type;
return true;
}
// The blobs of experts [0, ne) of layer l.
bool read_layer(const strata::GgufModel& m, int l, int ne, Layer& Y) {
const char* roles[3] = { "gate", "up", "down" };
const strata::TensorInfo* t[3] = {};
size_t sh[3] = {};
for (int r = 0; r < 3; ++r) t[r] = m.find("blk." + std::to_string(l) + ".ffn_" + roles[r] + "_exps.weight", &sh[r]);
if (!t[0] || !t[1] || !t[2]) return false;
std::string err;
if (!cpu::native_fmt((int) t[0]->type, (int) t[2]->type, kEmbd, kFf, Y.f, err)) {
std::printf("layer %d: %s\n", l, err.c_str());
return false;
}
const size_t gu_role = Y.f.up_off, dn_role = Y.f.bytes - Y.f.down_off;
Y.layer = l;
Y.ne = ne;
Y.host.resize((size_t) ne * Y.f.bytes);
const uint8_t* G = m.shard(sh[0]).tensor_data(*t[0]);
const uint8_t* U = m.shard(sh[1]).tensor_data(*t[1]);
const uint8_t* D = m.shard(sh[2]).tensor_data(*t[2]);
for (int e = 0; e < ne; ++e) {
uint8_t* b = Y.host.data() + (size_t) e * Y.f.bytes;
std::memcpy(b, G + (size_t) e * gu_role, gu_role);
std::memcpy(b + Y.f.up_off, U + (size_t) e * gu_role, gu_role);
std::memcpy(b + Y.f.down_off, D + (size_t) e * dn_role, dn_role);
}
return true;
}
// check (6): nibble::check_gguf over every slot (the start-up check), and nibble::gather_gguf (the prompt path's gather)
// in groups of 16 and from later first positions, against the blobs byte for byte (and the MMQ tail zeroed).
// `expect_diff`: the negative control (only the report says so: the counts must then be > 0).
long long gguf_parity(const Layer& Y, const unsigned long long* dp, bool expect_diff) {
long long bad_total = 0;
const int ne = Y.ne;
unsigned long long* dbad = nullptr;
CK(cudaMalloc(&dbad, 8));
CK(cudaMemset(dbad, 0, 8));
for (int e0 = 0; e0 < ne; e0 += 64) {
const int rc = nib::check_gguf(Y.V, Y.raw + (size_t) e0 * Y.V.raw_bytes, std::min(64, ne - e0), dp + e0, dbad, nullptr);
if (rc != 0) {
std::printf("check_gguf: %s\n", cudaGetErrorString((cudaError_t) rc));
std::exit(2);
}
}
CK(cudaDeviceSynchronize());
unsigned long long cbad = 0;
CK(cudaMemcpy(&cbad, dbad, 8, cudaMemcpyDeviceToHost));
CK(cudaFree(dbad));
// the gathers: MMQ's group buffers (16 experts of [gate | up] and of down, each followed by the 4096-byte tail)
const uint64_t gub = (uint64_t) nib::kRows * Y.f.gu_row, db = Y.V.down_bytes, zero = 4096;
uint8_t *ggu = nullptr, *gd = nullptr;
CK(cudaMalloc(&ggu, 16 * gub + zero));
CK(cudaMalloc(&gd, 16 * db + zero));
std::vector<uint8_t> hgu(16 * gub + zero), hd(16 * db + zero);
std::vector<unsigned long long> hp(ne);
CK(cudaMemcpy(hp.data(), dp, (size_t) ne * 8, cudaMemcpyDeviceToHost));
long long gbad = 0, groups = 0;
for (int e0 = 0; e0 < ne; e0 += 16) {
const int first = (e0 / 16) % 4 == 3 ? 1 + (e0 / 64) % 15 : 0; // every 4th group from a later position
const int n = std::min(16, ne - e0);
if (first >= n) continue;
const uint8_t* sl[16] = {};
for (int q = first; q < n; ++q) sl[q] = (const uint8_t*) (uintptr_t) hp[(size_t) (e0 + q)];
CK(cudaMemset(ggu, 0xA5, 16 * gub + zero));
CK(cudaMemset(gd, 0xA5, 16 * db + zero));
const int rc = nib::gather_gguf(Y.V, sl, first, n, ggu, gub, gd, db, zero, nullptr);
if (rc != 0) {
std::printf("gather_gguf: %s\n", cudaGetErrorString((cudaError_t) rc));
std::exit(2);
}
CK(cudaDeviceSynchronize());
CK(cudaMemcpy(hgu.data(), ggu, hgu.size(), cudaMemcpyDeviceToHost));
CK(cudaMemcpy(hd.data(), gd, hd.size(), cudaMemcpyDeviceToHost));
for (int q = first; q < n; ++q) {
const uint8_t* raw = Y.host.data() + (size_t) (e0 + q) * Y.f.bytes;
gbad += std::memcmp(hgu.data() + (size_t) q * gub, raw, gub) != 0;
gbad += std::memcmp(hd.data() + (size_t) q * db, raw + Y.f.down_off, db) != 0;
}
for (uint64_t i = 0; i < zero; ++i) gbad += hgu[(size_t) n * gub + i] != 0 || hd[(size_t) n * db + i] != 0 ? 1 : 0;
++groups;
}
CK(cudaFree(ggu));
CK(cudaFree(gd));
std::printf(" check_gguf: %llu words differ; gather_gguf (%lld groups, the tail zeroed): %lld matrices or tail bytes differ%s\n",
cbad, groups, gbad, expect_diff ? " (NEGATIVE CONTROL: both must be > 0)" : "");
bad_total += (long long) cbad + gbad; // the negative control fails the run, as (7) does: it must
return bad_total;
}
// Plans the layer, uploads the blobs, transcodes them into slots (checks 1-6).
long long convert(Layer& Y, bool ggml_check) {
long long bad_total = 0;
nib::Config c;
c.on = true;
Y.V = nib::plan_layer(Y.layer, Y.f, c);
Y.Lr = K::native_expert_layout(Y.f.gu_type, Y.f.d_type, kEmbd, kFf);
Y.Ln = K::native_expert_layout_nib(Y.f.gu_type, Y.f.d_type, kEmbd, kFf);
if (!Y.V.on || Y.V.slot_bytes != Y.Ln.bytes || Y.V.down_off != Y.Ln.down_off || Y.Lr.bytes != Y.f.bytes) {
std::printf("layer %d: plan_layer / native_expert_layout_nib disagree (on %d, slot %llu vs %zu)\n", Y.layer,
(int) Y.V.on, (unsigned long long) Y.V.slot_bytes, Y.Ln.bytes);
return 1;
}
const int ne = Y.ne;
const size_t slots_bytes = (size_t) (ne - 1) * Y.V.slot_bytes + Y.V.hashed;
CK(cudaMalloc(&Y.raw, Y.host.size()));
CK(cudaMemcpy(Y.raw, Y.host.data(), Y.host.size(), cudaMemcpyHostToDevice));
CK(cudaMalloc(&Y.slots, slots_bytes));
CK(cudaMemset(Y.slots, 0xA5, slots_bytes));
std::vector<unsigned long long> hp(ne);
for (int e = 0; e < ne; ++e) hp[e] = (unsigned long long) (Y.slots + (size_t) e * Y.V.slot_bytes);
unsigned long long *dp = nullptr, *dbad = nullptr, *dhash = nullptr;
CK(cudaMalloc(&dp, (size_t) ne * 8));
CK(cudaMalloc(&dbad, 8));
CK(cudaMalloc(&dhash, (size_t) ne * 8));
CK(cudaMemcpy(dp, hp.data(), (size_t) ne * 8, cudaMemcpyHostToDevice));
CK(cudaMemset(dbad, 0, 8));
// (1) the startup fill's batches: 64 blobs at a time, then the down rows behind the codes
for (int e0 = 0; e0 < ne; e0 += 64) {
const int n = std::min(64, ne - e0);
const int rc = nib::transcode(Y.V, Y.raw + (size_t) e0 * Y.V.raw_bytes, n, dp + e0, dbad, nullptr);
if (rc != 0) {
std::printf("transcode: %s\n", cudaGetErrorString((cudaError_t) rc));
std::exit(2);
}
}
for (int e = 0; e < ne; ++e)
CK(cudaMemcpyAsync(Y.slots + (size_t) e * Y.V.slot_bytes + nib::kDownOff,
Y.raw + (size_t) e * Y.V.raw_bytes + Y.V.raw_down_off, Y.V.down_bytes, cudaMemcpyDeviceToDevice));
CK(cudaDeviceSynchronize());
unsigned long long bad = 0;
CK(cudaMemcpy(&bad, dbad, 8, cudaMemcpyDeviceToHost));
std::printf("%s layer %2d %-7s / down %-6s: %d experts, blob %zu B, slot %llu B (x%.3f); transcode: %llu of %lld items do not round-trip\n",
Y.tag.c_str(), Y.layer, nib::fmt_name(Y.f.gu_type), ggml_type_name((ggml_type) Y.f.d_type), ne, Y.f.bytes,
(unsigned long long) Y.V.slot_bytes, (double) Y.V.slot_bytes / (double) Y.f.bytes, bad,
(long long) ne * nib::kRows * nib::kItems);
bad_total += (long long) bad;
bad_total += gguf_parity(Y, dp, false);
// (2) every gate/up byte written: the first 8 blobs again into slots pre-filled with 0x5A
{
const int n8 = std::min(8, ne);
uint8_t* alt = nullptr;
unsigned long long* ap = nullptr;
CK(cudaMalloc(&alt, (size_t) n8 * Y.V.slot_bytes));
CK(cudaMemset(alt, 0x5A, (size_t) n8 * Y.V.slot_bytes));
std::vector<unsigned long long> ah(n8);
for (int e = 0; e < n8; ++e) ah[e] = (unsigned long long) (alt + (size_t) e * Y.V.slot_bytes);
CK(cudaMalloc(&ap, (size_t) n8 * 8));
CK(cudaMemcpy(ap, ah.data(), (size_t) n8 * 8, cudaMemcpyHostToDevice));
CK(cudaMemset(dbad, 0, 8));
if (nib::transcode(Y.V, Y.raw, n8, ap, dbad, nullptr) != 0) std::exit(2);
CK(cudaDeviceSynchronize());
std::vector<uint8_t> a(nib::kGuBytes), b(nib::kGuBytes);
long long wbad = 0;
for (int e = 0; e < n8; ++e) {
CK(cudaMemcpy(a.data(), Y.slots + (size_t) e * Y.V.slot_bytes, nib::kGuBytes, cudaMemcpyDeviceToHost));
CK(cudaMemcpy(b.data(), alt + (size_t) e * Y.V.slot_bytes, nib::kGuBytes, cudaMemcpyDeviceToHost));
wbad += std::memcmp(a.data(), b.data(), nib::kGuBytes) != 0;
}
std::printf(" every gate/up byte written (0xA5 vs 0x5A pre-fill, %d slots): %lld differ\n", n8, wbad);
bad_total += wbad;
CK(cudaFree(alt));
CK(cudaFree(ap));
}
// (3) the device hash of every slot against the host reference over the slot read back
const int rc = nib::hash_slots(Y.V, dp, ne, dhash, nullptr);
if (rc != 0) {
std::printf("hash_slots: %s\n", cudaGetErrorString((cudaError_t) rc));
std::exit(2);
}
std::vector<unsigned long long> hh(ne);
CK(cudaMemcpy(hh.data(), dhash, (size_t) ne * 8, cudaMemcpyDeviceToHost));
std::vector<uint8_t> sl(slots_bytes);
CK(cudaMemcpy(sl.data(), Y.slots, slots_bytes, cudaMemcpyDeviceToHost));
std::atomic<long long> hbad{ 0 }, cbad{ 0 }, dnbad{ 0 }, gbad{ 0 };
std::atomic<long long> grows{ 0 };
std::string first_ggml;
std::atomic<int> first_set{ 0 };
const ggml_type_traits* tr = ggml_get_type_traits((ggml_type) Y.f.gu_type);
const unsigned nth = std::max(1u, std::min(16u, std::thread::hardware_concurrency()));
std::vector<std::thread> th;
for (unsigned ti = 0; ti < nth; ++ti)
th.emplace_back([&, ti]() {
std::vector<float> ref((size_t) kEmbd), got((size_t) kEmbd);
for (int e = (int) ti; e < ne; e += (int) nth) {
const uint8_t* s = sl.data() + (size_t) e * Y.V.slot_bytes;
const uint8_t* raw = Y.host.data() + (size_t) e * Y.f.bytes;
hbad += nib::hash_host(s, Y.V.hashed) != hh[e];
if (e == 0) { // the hash sees every byte: one flipped must change it
std::vector<uint8_t> t2(s, s + Y.V.hashed);
t2[Y.V.hashed - 1] ^= 1;
hbad += nib::hash_host(t2.data(), Y.V.hashed) == hh[e];
}
dnbad += std::memcmp(s + nib::kDownOff, raw + Y.V.raw_down_off, Y.V.down_bytes) != 0;
for (int64_t r = 0; r < nib::kRows; ++r) {
// (4) the host codec
for (int64_t k = 0; k < nib::kItems; ++k) {
uint32_t q[4], q2[4];
int8_t g[32];
std::memcpy(q, s + nib::code_off(r, k), 16);
nib::decode_item(Y.f.gu_type, q, g);
const bool ok = nib::encode_item(Y.f.gu_type, g, q2);
cbad += !ok || std::memcmp(q, q2, 16) != 0;
}
if (!ggml_check) continue;
// (5) ggml's dequantizer of the GGUF row against the slot's weights through its expression
const uint8_t* row = raw + (r < kFf ? (size_t) r * Y.f.gu_row : Y.f.up_off + (size_t) (r - kFf) * Y.f.gu_row);
tr->to_float(row, ref.data(), kEmbd);
for (int64_t k = 0; k < nib::kItems; ++k) {
uint32_t q[4];
std::memcpy(q, s + nib::code_off(r, k), 16);
int8_t gv[32];
nib::decode_item(Y.f.gu_type, q, gv);
const int sc = s[nib::scale_off(r, k)];
ggml_fp16_t dh;
std::memcpy(&dh, s + nib::d_off(r, k / 8), sizeof dh);
const float d = ggml_fp16_to_fp32(dh);
for (int j = 0; j < 32; ++j) {
const float db = Y.f.gu_type == nib::kIQ2_S ? d * (0.5f + ((j < 16 ? sc : sc >> 4) & 0xf)) * 0.25f
: Y.f.gu_type == nib::kIQ3_XXS ? d * (0.5f + sc) * 0.5f
: d * sc;
const int a = gv[j] < 0 ? -(int) gv[j] : (int) gv[j];
float v = db * a;
if (gv[j] < 0) v = -v;
got[(size_t) (k * 32 + j)] = v;
}
}
++grows;
if (std::memcmp(ref.data(), got.data(), ref.size() * sizeof(float)) != 0) {
++gbad;
if (first_set.exchange(1) == 0) {
size_t cc = 0;
while (cc < ref.size() && std::memcmp(&ref[cc], &got[cc], 4) == 0) ++cc;
char buf[200];
std::snprintf(buf, sizeof buf, "expert %d row %lld column %zu: slot %.9g, GGUF %.9g", e,
(long long) r, cc, (double) got[cc], (double) ref[cc]);
first_ggml = buf;
}
}
}
}
});
for (auto& t : th) t.join();
std::printf(" hash_slots vs hash_host: %lld of %d slots differ (+1 flipped-byte check); down rows: %lld of %d slots differ;"
" host codec: %lld of %lld items differ\n",
hbad.load(), ne, dnbad.load(), ne, cbad.load(), (long long) ne * nib::kRows * nib::kItems);
if (ggml_check)
std::printf(" ggml to_float of the GGUF rows vs the slot's weights: %lld of %lld rows differ%s%s\n", gbad.load(),
grows.load(), first_ggml.empty() ? "" : "; first: ", first_ggml.c_str());
bad_total += hbad + dnbad + cbad + gbad;
CK(cudaFree(dp));
CK(cudaFree(dbad));
CK(cudaFree(dhash));
return bad_total;
}
// ---- plans: synthetic top-10 routing and resident_plan_kernel's grouping (groups by first appearance)
struct Plan {
int ng = 0, ne = 0, ntok = 0, maxn = 0;
std::vector<int> gid, start, tok, dst;
};
void make_routing(std::mt19937& rng, int NE, int T, int share_pct, std::vector<int>& ids) {
ids.clear();
std::vector<int> prev, cur;
for (int t = 0; t < T; ++t) {
cur.clear();
const bool sh = t > 0 && share_pct > 0;
if (sh) {
std::vector<int> p = prev;
std::shuffle(p.begin(), p.end(), rng);
cur.assign(p.begin(), p.begin() + 10 * share_pct / 100);
}
while ((int) cur.size() < 10) {
const int e = (int) (rng() % (unsigned) NE);
if (std::find(cur.begin(), cur.end(), e) != cur.end()) continue;
if (sh && std::find(prev.begin(), prev.end(), e) != prev.end()) continue;
cur.push_back(e);
}
std::shuffle(cur.begin(), cur.end(), rng);
ids.insert(ids.end(), cur.begin(), cur.end());
prev = cur;
}
}
Plan build_plan(const std::vector<int>& ids) {
const int n = (int) ids.size();
Plan p;
std::vector<int> first(n), cnt, grp_of(n, -1);
for (int j = 0; j < n; ++j) {
int f = j;
for (int i = 0; i < j; ++i)
if (ids[i] == ids[j]) { f = i; break; }
first[j] = f;
}
for (int j = 0; j < n; ++j)
if (first[j] == j) {
grp_of[j] = p.ng++;
p.gid.push_back(ids[j]);
cnt.push_back(0);
}
for (int j = 0; j < n; ++j) cnt[grp_of[first[j]]]++;
p.start.assign(p.ng + 1, 0);
int acc = 0;
for (int g = 0; g < p.ng; ++g) {
p.start[g] = acc;
acc += cnt[g];
p.maxn = std::max(p.maxn, cnt[g]);
}
p.start[p.ng] = acc;
p.tok.assign(n, 0);
p.dst.assign(n, 0);
std::vector<int> fill(p.ng, 0);
for (int j = 0; j < n; ++j) {
const int g = grp_of[first[j]];
const int at = p.start[g] + fill[g]++;
p.tok[at] = j / 10;
p.dst[at] = j;
}
p.ne = n;
p.ntok = n / 10;
return p;
}
// one group of T entries (G = 1): an expert every token of the window routes to
Plan single_group(std::mt19937& rng, int NE, int T) {
Plan p;
p.ntok = T;
p.ng = 1;
p.gid.push_back((int) (rng() % (unsigned) NE));
p.start = { 0, T };
for (int t = 0; t < T; ++t) {
p.tok.push_back(t);
p.dst.push_back(T - 1 - t);
}
p.ne = T;
p.maxn = T;
return p;
}
// groups of 0..11 entries (more than one GRP_NC pass, empty groups), 12 tokens, distinct tokens within a group,
// destination rows in any order
Plan synthetic_plan(std::mt19937& rng, int NE) {
Plan p;
p.ntok = kTok;
std::vector<int> ids(NE);
for (int i = 0; i < NE; ++i) ids[i] = i;
std::shuffle(ids.begin(), ids.end(), rng);
const int sizes[] = { 3, 0, 11, 1, 5, 8, 0, 2, 4, 9, 6, 7, 10, 1 };
int acc = 0;
for (int g = 0; g < (int) (sizeof sizes / sizeof sizes[0]); ++g) {
p.gid.push_back(ids[(size_t) g % ids.size()]);
p.start.push_back(acc);
std::vector<int> toks(kTok);
for (int i = 0; i < kTok; ++i) toks[i] = i;
std::shuffle(toks.begin(), toks.end(), rng);
std::sort(toks.begin(), toks.begin() + sizes[g]);
for (int i = 0; i < sizes[g]; ++i) {
p.tok.push_back(toks[i]);
p.dst.push_back(acc + i);
}
acc += sizes[g];
p.maxn = std::max(p.maxn, sizes[g]);
}
p.start.push_back(acc);
p.ng = (int) p.gid.size();
p.ne = acc;
std::vector<int> perm(acc);
for (int i = 0; i < acc; ++i) perm[i] = i;
std::shuffle(perm.begin(), perm.end(), rng);
for (int i = 0; i < acc; ++i) p.dst[i] = perm[i];
return p;
}
std::vector<Q81> make_x(std::mt19937& rng) {
std::vector<Q81> x((size_t) kTok * (kEmbd / 32));
for (size_t i = 0; i < x.size(); ++i) {
Q81& b = x[i];
const int tok = (int) (i / (kEmbd / 32));
const float d = 0.002f + 0.03f * (float) (rng() % 1000) / 1000.0f;
int sum = 0;
for (int j = 0; j < 32; ++j) {
int q = (int) (rng() % 255) - 127;
if (tok == 1) q = 127; // the sum of x at its extremes (+-4064)
if (tok == 2) q = -127;
if (tok == 3 && (i % (kEmbd / 32)) < 8) q = (j & 1) ? 127 : -127;
b.qs[j] = (int8_t) q;
sum += q;
}
b.d = ggml_fp32_to_fp16(d);
b.s = ggml_fp32_to_fp16(d * (float) sum);
}
return x;
}
struct Dev {
unsigned long long* ptr = nullptr;
int32_t *start = nullptr, *ng = nullptr, *tok = nullptr, *dst = nullptr;
};
Dev upload(const Plan& p, const Layer& Y, bool conv) {
Dev d;
std::vector<unsigned long long> hp(std::max(1, p.ng));
for (int g = 0; g < p.ng; ++g)
hp[g] = conv ? (unsigned long long) (Y.slots + (size_t) p.gid[g] * Y.V.slot_bytes)
: (unsigned long long) (Y.raw + (size_t) p.gid[g] * Y.f.bytes);
CK(cudaMalloc(&d.ptr, (size_t) std::max(1, p.ng) * 8));
CK(cudaMalloc(&d.start, (size_t) (p.ng + 1) * 4));
CK(cudaMalloc(&d.ng, 4));
CK(cudaMalloc(&d.tok, (size_t) std::max(1, p.ne) * 4));
CK(cudaMalloc(&d.dst, (size_t) std::max(1, p.ne) * 4));
CK(cudaMemcpy(d.ptr, hp.data(), hp.size() * 8, cudaMemcpyHostToDevice));
CK(cudaMemcpy(d.start, p.start.data(), (size_t) (p.ng + 1) * 4, cudaMemcpyHostToDevice));
CK(cudaMemcpy(d.ng, &p.ng, 4, cudaMemcpyHostToDevice));
if (p.ne) {
CK(cudaMemcpy(d.tok, p.tok.data(), (size_t) p.ne * 4, cudaMemcpyHostToDevice));
CK(cudaMemcpy(d.dst, p.dst.data(), (size_t) p.ne * 4, cudaMemcpyHostToDevice));
}
return d;
}
void release(Dev& d) {
CK(cudaFree(d.ptr));
CK(cudaFree(d.start));
CK(cudaFree(d.ng));
CK(cudaFree(d.tok));
CK(cudaFree(d.dst));
}
struct Outs {
std::vector<uint8_t> gate, up, h, hq, out;
};
// one native_expert_grouped call; scratch and out pre-filled with `fill` so an unwritten value shows
Outs run(const Layer& Y, const Plan& p, bool conv, const void* dx, void* scr, float* dout, int64_t gy, int mode,
uint8_t fill) {
// the window's capacity, as verify sizes it: T x 10 groups and entries (the blocks past the plan's groups idle)
const int64_t cap_e = std::max<int64_t>((int64_t) p.ntok * 10, std::max(p.ne, p.ng)), cap_g = cap_e;
Dev d = upload(p, Y, conv);
CK(cudaMemset(scr, fill, K::native_expert_scratch_bytes(cap_e, kFf)));
CK(cudaMemset(dout, fill, (size_t) cap_e * kEmbd * 4));
K::native_expert_set_mode(mode, 0);
K::native_expert_grouped(conv ? Y.Ln : Y.Lr, d.ptr, d.start, d.ng, d.dst, d.tok, cap_g, cap_e, dx, scr, dout, nullptr, gy);
K::native_expert_set_mode(-1, 0);
CK(cudaGetLastError());
CK(cudaDeviceSynchronize());
const size_t f = (size_t) cap_e * kFf * 4, fa = (f + 255) & ~(size_t) 255;
Outs o;
o.gate.resize((size_t) p.ne * kFf * 4);
o.up.resize(o.gate.size());
o.h.resize(o.gate.size());
o.hq.resize((size_t) p.ne * (kFf / 32) * 36);
o.out.resize((size_t) p.ne * kEmbd * 4);
const uint8_t* s = (const uint8_t*) scr;
if (p.ne) {
CK(cudaMemcpy(o.gate.data(), s, o.gate.size(), cudaMemcpyDeviceToHost));
CK(cudaMemcpy(o.up.data(), s + fa, o.up.size(), cudaMemcpyDeviceToHost));
CK(cudaMemcpy(o.h.data(), s + 2 * fa, o.h.size(), cudaMemcpyDeviceToHost));
CK(cudaMemcpy(o.hq.data(), s + 3 * fa, o.hq.size(), cudaMemcpyDeviceToHost));
std::vector<uint8_t> all((size_t) cap_e * kEmbd * 4);
CK(cudaMemcpy(all.data(), dout, all.size(), cudaMemcpyDeviceToHost));
for (int e = 0; e < p.ne; ++e)
std::memcpy(o.out.data() + (size_t) e * kEmbd * 4, all.data() + (size_t) p.dst[e] * kEmbd * 4, kEmbd * 4);
}
release(d);
return o;
}
long long diff_rows(const std::vector<uint8_t>& a, const std::vector<uint8_t>& b, size_t row) {
long long n = 0;
for (size_t i = 0; i + row <= a.size(); i += row) n += std::memcmp(a.data() + i, b.data() + i, row) != 0;
return n;
}
// check (7): every plan, both grid strides, every kernel choice, both layouts, v1 off / on, against the GGUF blobs
// through the production kernels
long long grouped_parity(const Layer& Y, std::mt19937& rng, int plans, const void* dx, void* scr, float* dout) {
struct Case {
Plan p;
int T;
};
std::vector<Case> cases;
for (int T = 1; T <= 6; ++T) {
for (int share : { 0, 50 })
for (int i = 0; i < plans; ++i) {
std::vector<int> ids;
make_routing(rng, Y.ne, T, share, ids);
cases.push_back({ build_plan(ids), T });
}
cases.push_back({ single_group(rng, Y.ne, T), T });
}
for (int i = 0; i < 4; ++i) cases.push_back({ synthetic_plan(rng, Y.ne), kTok });
int hist[13] = {};
for (const Case& cs : cases)
for (int g = 0; g < cs.p.ng; ++g) hist[std::min(12, cs.p.start[g + 1] - cs.p.start[g])]++;
struct Var {
int v, mode;
bool v1;
const char* name;
};
// mode -1: the card's default layout (a GP100: mode 8 - its reference gate/up kernel writes hq, not gate / up);
// mode 0: the CUDA layout
const Var vars[] = { { -1, -1, false, "default, card" }, { 0, -1, false, "S, card" }, { 1, -1, false, "P, card" },
{ -1, -1, true, "default, card, V1" }, { -1, 0, false, "default, mode 0" },
{ 0, 0, false, "S, mode 0" }, { 1, 0, false, "P, mode 0" }, { -1, 0, true, "default, mode 0, V1" },
// STRATA_NIB_GU_PROTO (nib_gu_proto.cuh): the staged-once kernel, id 31
{ 31, -1, false, "gu-proto 31, card" }, { 31, 0, false, "gu-proto 31, mode 0" },
{ 31, 0, true, "gu-proto 31, mode 0, V1" } };
constexpr int NV = (int) (sizeof vars / sizeof vars[0]);
long long rows[NV] = {}, bad[NV][5] = {}, total = 0;
for (const Case& cs : cases)
for (int64_t gy : { (int64_t) 0, (int64_t) 4 }) {
Outs ref[2][2]; // [mode 0][v1]
bool have[2][2] = {};
for (int vi = 0; vi < NV; ++vi) {
const int m0 = vars[vi].mode == 0 ? 1 : 0, r1 = vars[vi].v1 ? 1 : 0;
K::native_grouped_set_v1(vars[vi].v1);
if (!have[m0][r1]) {
K::native_expert_nib_set_variant(-1);
ref[m0][r1] = run(Y, cs.p, false, dx, scr, dout, gy, vars[vi].mode, 0xFF);
have[m0][r1] = true;
}
K::native_expert_nib_set_variant(vars[vi].v);
const Outs o = run(Y, cs.p, true, dx, scr, dout, gy, vars[vi].mode, 0x00);
rows[vi] += cs.p.ne;
// gate / up: written by both arms in the CUDA layout, and by every arm of a striding call (gy 4: mode 0)
const bool gu = vars[vi].mode == 0 || gy == 4;
if (gu) {
bad[vi][0] += diff_rows(ref[m0][r1].gate, o.gate, kFf * 4);
bad[vi][1] += diff_rows(ref[m0][r1].up, o.up, kFf * 4);
}
if (vars[vi].v1 && gu) bad[vi][2] += diff_rows(ref[m0][r1].h, o.h, kFf * 4);
bad[vi][3] += diff_rows(ref[m0][r1].hq, o.hq, (kFf / 32) * 36);
bad[vi][4] += diff_rows(ref[m0][r1].out, o.out, kEmbd * 4);
}
}
K::native_expert_nib_set_variant(-1);
K::native_grouped_set_v1(false);
std::printf(" plans: %zu (T 1..6 x sharing 0/50%% x %d, G=1 per T, 4 of 0..11 entries); groups by entries:", cases.size(),
plans);
for (int n = 0; n <= 12; ++n) std::printf(" %d:%d", n, hist[n]);
std::printf("\n");
for (int vi = 0; vi < NV; ++vi) {
const long long b = bad[vi][0] + bad[vi][1] + bad[vi][2] + bad[vi][3] + bad[vi][4];
std::printf(" native_expert_grouped %-20s: %6lld entries x 2 strides, rows differing: gate %lld, up %lld, h %s, hq %lld, out %lld\n",
vars[vi].name, rows[vi], bad[vi][0], bad[vi][1],
vars[vi].v1 ? std::to_string(bad[vi][2]).c_str() : "-", bad[vi][3], bad[vi][4]);
total += b;
}
return total;
}
// --synthetic: random gate/up blocks, d cycling through the fp16 extremes; the coverage of each field is counted
long long synthesize(Layer& Y, std::mt19937& rng) {
static const uint16_t kD[] = { 0x0001, 0x8001, 0x7BFF, 0xFBFF, 0x3C00, 0xBC00, 0x0400, 0x1A00, 0x2400 };
const int gt = Y.f.gu_type;
const size_t bb = gt == nib::kIQ2_S ? 82 : gt == nib::kIQ3_XXS ? 98 : 110; // block_iq2_s / iq3_xxs / iq3_s
const size_t nblk = (size_t) nib::kRows * nib::kSuper;
std::set<int> grid, sign, scale, dset;
for (int e = 0; e < Y.ne; ++e) {
uint8_t* b = Y.host.data() + (size_t) e * Y.f.bytes;
for (size_t i = 0; i < 2 * Y.f.up_off; ++i) b[i] = (uint8_t) rng();
for (size_t k = 0; k < nblk; ++k) {
uint8_t* blk = b + k * bb;
const uint16_t d = kD[((size_t) e * nblk + k) % (sizeof kD / sizeof kD[0])];
std::memcpy(blk, &d, 2);
dset.insert(d);
const uint8_t* qs = blk + 2;
for (int ib = 0; ib < 8; ++ib) {
if (gt == nib::kIQ2_S) {
const uint8_t* qh = blk + 66;
const uint8_t* sc = blk + 74;
for (int l = 0; l < 4; ++l) {
grid.insert(qs[4 * ib + l] | ((qh[ib] << (8 - 2 * l)) & 0x300));
sign.insert(qs[32 + 4 * ib + l]);
}
scale.insert(sc[ib] & 15);
scale.insert(sc[ib] >> 4);
} else if (gt == nib::kIQ3_XXS) {
uint32_t aux;
std::memcpy(&aux, blk + 66 + 4 * ib, 4);
for (int l = 0; l < 8; ++l) grid.insert(qs[8 * ib + l]);
for (int l = 0; l < 4; ++l) sign.insert((int) ((aux >> (7 * l)) & 127));
scale.insert((int) (aux >> 28));
} else {
const uint8_t qh = blk[66 + ib];
const uint8_t* sg = blk + 74;
const uint8_t* sc = blk + 106;
for (int l = 0; l < 4; ++l) {
grid.insert(qs[8 * ib + 2 * l] | ((qh << (8 - 2 * l)) & 256));
grid.insert(qs[8 * ib + 2 * l + 1] | ((qh << (7 - 2 * l)) & 256));
sign.insert(sg[4 * ib + l]);
}
scale.insert((sc[ib / 2] >> (4 * (ib & 1))) & 15);
}
}
}
}
const size_t want_grid = gt == nib::kIQ2_S ? 1024 : gt == nib::kIQ3_XXS ? 256 : 512;
const size_t want_sign = gt == nib::kIQ3_XXS ? 128 : 256;
const bool ok = grid.size() == want_grid && sign.size() == want_sign && scale.size() == 16 &&
dset.size() == sizeof kD / sizeof kD[0];
std::printf("synthetic %-7s: %d experts, coverage: grid indices %zu/%zu, sign %s %zu/%zu, scale nibbles %zu/16, d values %zu/%zu"
" (+-min subnormal, +-65504, +-1, min normal, 2 ordinary) %s\n",
nib::fmt_name(gt), Y.ne, grid.size(), want_grid, gt == nib::kIQ3_XXS ? "indices" : "bytes", sign.size(),
want_sign, scale.size(), dset.size(), sizeof kD / sizeof kD[0], ok ? "complete" : "INCOMPLETE");
return ok ? 0 : 1;
}
std::vector<int> split_ints(const std::string& s) {
std::vector<int> v;
size_t p = 0;
while (p < s.size()) {
v.push_back(std::atoi(s.c_str() + p));
p = s.find(',', p);
if (p == std::string::npos) break;
++p;
}
return v;
}
} // namespace
int main(int argc, char** argv) {
setvbuf(stdout, nullptr, _IONBF, 0);
if (argc == 2 && std::strcmp(argv[1], "--header") == 0) return header_check();
std::vector<std::string> shards;
int per_fmt = 2, NE = 512, plans = 8, syn = 64;
bool ggml_check = true, negative = false, syn_only = false;
std::vector<int> only;
for (int i = 1; i < argc; ++i) {
const std::string a = argv[i];
auto next = [&]() -> const char* {
if (i + 1 >= argc) {
std::fprintf(stderr, "%s needs a value\n", a.c_str());
std::exit(2);
}
return argv[++i];
};
if (a == "--layers-per-fmt") per_fmt = std::atoi(next());
else if (a == "--layers") only = split_ints(next());
else if (a == "--ne") NE = std::atoi(next());
else if (a == "--plans") plans = std::atoi(next());
else if (a == "--synthetic") syn = std::atoi(next());
else if (a == "--no-ggml") ggml_check = false;
else if (a == "--negative") negative = true;
else if (a == "--synthetic-only") syn_only = true;
else if (a.size() > 5 && a.substr(a.size() - 5) == ".gguf") shards.push_back(a);
else {
std::fprintf(stderr,
"usage: iq_nibble_parity --header\n"
" iq_nibble_parity <shard.gguf> [<shard.gguf> ...] [--layers-per-fmt 2] [--layers 1,17] [--ne 512]\n"
" [--plans 8] [--synthetic 64] [--no-ggml] [--negative]\n"
" iq_nibble_parity --synthetic-only [--synthetic 64] [--plans 8] [--no-ggml]\n");
return 2;
}
}
if ((shards.empty() && !syn_only) || NE < 16 || NE > 512) {
std::fprintf(stderr, "iq_nibble_parity: give at least one model shard (.gguf); --ne 16..512\n");
return 2;
}
CK(cudaSetDevice(0));
cudaDeviceProp prop{};
CK(cudaGetDeviceProperties(&prop, 0));
char bus[64] = {};
CK(cudaDeviceGetPCIBusId(bus, sizeof bus, 0));
std::printf("device 0: %s, bus %s, cc %d.%d, %d SMs\n", prop.name, bus, prop.major, prop.minor, prop.multiProcessorCount);
if (prop.major != 6 || prop.minor != 0) {
std::fprintf(stderr, "iq_nibble_parity: the 4-bit kernels are the sm_60 build's; this GPU is cc %d.%d (skipped)\n",
prop.major, prop.minor);
return 77;
}
if (!STRATA_EXPERT_NIBBLE_BUILT || nib::prepare(nullptr) != 0) {
std::fprintf(stderr, "iq_nibble_parity: no 4-bit kernels in this build, or the GGUF codebooks are not unique\n");
return 2;
}
std::mt19937 rng(20261019);
void* scr = nullptr;
float* dout = nullptr;
void* dx = nullptr;
const int64_t capmax = (int64_t) kTok * 10;
CK(cudaMalloc(&scr, K::native_expert_scratch_bytes(capmax, kFf)));
CK(cudaMalloc((void**) &dout, (size_t) capmax * kEmbd * 4));
const std::vector<Q81> hx = make_x(rng);
CK(cudaMalloc(&dx, hx.size() * sizeof(Q81)));
CK(cudaMemcpy(dx, hx.data(), hx.size() * sizeof(Q81), cudaMemcpyHostToDevice));
if (syn_only) { // no model: every gate/up format with each down type, random experts
long long total_s = 0;
int done = 0;
for (const int gt : {nib::kIQ2_S, nib::kIQ3_XXS, nib::kIQ3_S})
for (const int dt : {20, 42}) { // the packs' down types: IQ4_NL, Q2_0
Layer S;
std::string err;
if (!cpu::native_fmt(gt, dt, kEmbd, kFf, S.f, err)) {
std::printf("%s / down type %d: %s\n", nib::fmt_name(gt), dt, err.c_str());
++total_s;
continue;
}
S.tag = "syn";
S.layer = 0;
S.ne = std::max(16, std::min(syn, NE));
S.host.resize((size_t) S.ne * S.f.bytes);
for (auto& b : S.host) b = (uint8_t) rng();
// block_iq4_nl and block_q2_0 are 18 bytes, the fp16 scale first: an ordinary scale in every block of the down rows
for (int e = 0; e < S.ne; ++e)
for (size_t o = S.f.down_off; o + 18 <= S.f.bytes; o += 18) {
const uint16_t sc = (uint16_t) ((9u << 10) | (rng() & 0x3FFu));
std::memcpy(S.host.data() + (size_t) e * S.f.bytes + o, &sc, 2);
}
long long sb = synthesize(S, rng);
sb += convert(S, ggml_check);
sb += grouped_parity(S, rng, plans, dx, scr, dout);
total_s += sb;
++done;
CK(cudaFree(S.raw));
CK(cudaFree(S.slots));
}
std::printf("\n%s: %d synthetic format pairs, %lld differences\n", total_s == 0 && done == 6 ? "PASS" : "FAIL", done, total_s);
return total_s == 0 && done == 6 ? 0 : 1;
}
long long total = 0;
int layers_done = 0;
std::vector<int> syn_done;
for (size_t si = 0; si < shards.size(); ++si) {
std::unique_ptr<strata::GgufModel> model;
try {
model = std::make_unique<strata::GgufModel>(strata::GgufModel::open(shards[si]));
} catch (const std::exception& ex) {
std::fprintf(stderr, "iq_nibble_parity: %s\n", ex.what());
return 2;
}
const std::string tag(1, (char) ('A' + si));
std::printf("\n#### model %s: %s (%zu shards)\n", tag.c_str(), shards[si].c_str(), model->size());
std::vector<int> pick;
int count[64] = {};
for (int l = 0; l < 256; ++l) {
int gt = -1, dt = -1;
if (!layer_fmt(*model, l, gt, dt)) break;
if (!nib::fmt_built(gt)) continue;
const bool want = only.empty() ? count[gt]++ < per_fmt : std::find(only.begin(), only.end(), l) != only.end();
if (want) pick.push_back(l);
}
for (int l : pick) {
Layer Y;
Y.tag = tag;
if (!read_layer(*model, l, NE, Y)) {
std::printf("layer %d: cannot read its experts\n", l);
++total;
continue;
}
long long b = convert(Y, ggml_check);
if (negative) { // the control: one code byte changed per slot (item 3 of row 5), which (6) must see
for (int e = 0; e < Y.ne; ++e) {
uint8_t* at = Y.slots + (size_t) e * Y.V.slot_bytes + nib::code_off(5, 3);
uint8_t v = 0;
CK(cudaMemcpy(&v, at, 1, cudaMemcpyDeviceToHost));
v ^= 0x11;
CK(cudaMemcpy(at, &v, 1, cudaMemcpyHostToDevice));
}
std::printf(" NEGATIVE CONTROL: one code byte changed per slot - the rows below must differ\n");
std::vector<unsigned long long> hp(Y.ne);
for (int e = 0; e < Y.ne; ++e) hp[e] = (unsigned long long) (Y.slots + (size_t) e * Y.V.slot_bytes);
unsigned long long* dp = nullptr;
CK(cudaMalloc(&dp, (size_t) Y.ne * 8));
CK(cudaMemcpy(dp, hp.data(), (size_t) Y.ne * 8, cudaMemcpyHostToDevice));
b += gguf_parity(Y, dp, true);
CK(cudaFree(dp));
}
b += grouped_parity(Y, rng, plans, dx, scr, dout);
total += b;
++layers_done;
// the synthetic experts of this format, once: this layer's blobs with random gate/up blocks
if (syn > 0 && std::find(syn_done.begin(), syn_done.end(), Y.f.gu_type) == syn_done.end() && !negative) {
syn_done.push_back(Y.f.gu_type);
Layer S;
S.tag = "syn";
S.layer = Y.layer;
S.f = Y.f;
S.ne = std::min(syn, Y.ne);
S.host.assign(Y.host.begin(), Y.host.begin() + (ptrdiff_t) ((size_t) S.ne * Y.f.bytes));
long long sb = synthesize(S, rng);
sb += convert(S, ggml_check);
sb += grouped_parity(S, rng, std::max(1, plans / 2), dx, scr, dout);
total += sb;
CK(cudaFree(S.raw));
CK(cudaFree(S.slots));
}
CK(cudaFree(Y.raw));
CK(cudaFree(Y.slots));
}
}
std::printf("\n%s: %d layers (+ %zu synthetic formats), %lld differences\n", total == 0 ? "PASS" : "FAIL", layers_done,
syn_done.size(), total);
return total == 0 ? 0 : 1;
}