Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// Q5_1 shared-expert projection: Strata CUDA dequantization vs ggml.
#define NOMINMAX
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/artifact/gguf_reader.hpp"
#include "strata/kernels/dequant_bf16.hpp"
#include "ggml.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <vector>
int main(int argc, char **argv) try {
if (argc != 2) {
std::fprintf(stderr, "usage: q5_projection_parity <shard1.gguf>\n");
return 2;
}
strata::GgufFile gguf(argv[1]);
const auto* t = gguf.find("blk.0.ffn_down_shexp.weight");
if (!t || t->type != 7 || t->shape.size() != 2) {
std::fprintf(stderr, "expected blk.0.ffn_down_shexp.weight Q5_1\n");
return 2;
}
const int64_t cols = (int64_t) t->shape[0], rows = std::min<int64_t>(4, (int64_t) t->shape[1]);
const size_t weight_bytes = (size_t) rows * (size_t) (cols / 32) * 24;
const size_t elements = (size_t) rows * (size_t) cols;
const uint8_t* source = gguf.tensor_data(*t);
void *dweight = nullptr, *dfloat = nullptr, *dbf16 = nullptr;
if (DPCT_CHECK_ERROR(dweight = (void *)sycl::malloc_device(
weight_bytes, dpct::get_in_order_queue())) != 0 ||
DPCT_CHECK_ERROR(
dfloat = (void *)sycl::malloc_device(
elements * sizeof(float), dpct::get_in_order_queue())) != 0 ||
DPCT_CHECK_ERROR(dbf16 = (void *)sycl::malloc_device(
elements * sizeof(uint16_t),
dpct::get_in_order_queue())) != 0) {
std::fprintf(stderr, "CUDA allocation failed\n");
return 1;
}
/*
DPCT1114: cudaMemcpy is migrated to asynchronization memcpy, assuming in
the original code the source host memory is pageable memory. If the memory
is not pageable, call wait() on event return by memcpy API to ensure
synchronization behavior.
*/
(dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue()).memcpy(dweight, source, weight_bytes).wait();
strata::kernels::dequant_f32(7, dweight, 0, rows, cols, (float*) dfloat, nullptr);
strata::kernels::dequant_bf16(7, dweight, 0, rows, cols, (uint16_t*) dbf16, nullptr);
std::vector<float> got(elements), ref(elements);
std::vector<uint16_t> got_bf16(elements);
(dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
.memcpy(got.data(), dfloat, elements * sizeof(float))
.wait();
(dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
.memcpy(got_bf16.data(), dbf16, elements * sizeof(uint16_t))
.wait();
sycl::free(dweight, dpct::get_in_order_queue());
sycl::free(dfloat, dpct::get_in_order_queue());
sycl::free(dbf16, dpct::get_in_order_queue());
const auto* traits = ggml_get_type_traits(GGML_TYPE_Q5_1);
for (int64_t r = 0; r < rows; ++r)
traits->to_float(source + (size_t) r * (size_t) (cols / 32) * 24,
ref.data() + (size_t) r * (size_t) cols, cols);
double max_abs = 0;
size_t bf16_bad = 0;
for (size_t i = 0; i < elements; ++i) {
max_abs = std::max(max_abs, (double) std::fabs(got[i] - ref[i]));
uint32_t bits;
std::memcpy(&bits, &ref[i], sizeof(bits));
const uint16_t expected = (uint16_t) ((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16);
if (got_bf16[i] != expected) ++bf16_bad;
}
const bool ok = max_abs <= 1e-6 && bf16_bad == 0;
std::printf("Q5_1 projection: %zu elements, max_abs %.3e, bf16 differences %zu %s\n",
elements, max_abs, bf16_bad, ok ? "PASS" : "FAIL");
return ok ? 0 : 1;
}
catch (sycl::exception const &exc) {
std::cerr << exc.what() << "Exception caught at file:" << __FILE__
<< ", line:" << __LINE__ << std::endl;
std::exit(1);
}