Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Metal: Gyro read-ahead skips the n-gram table, Q6_K/IQ4_NL hyper-connection matrices go through BF16 before their Q8_0 copy (as the pack does for CUDA)
Niko12211b47c1d076a3
diff --git a/metal/src/engine/window.cpp b/metal/src/engine/window.cppindex db875e8b6..3acd4a25c 100644--- a/metal/src/engine/window.cpp+++ b/metal/src/engine/window.cpp@@ -455,7 +455,18 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionStast_ = std::make_unique<ExpertStream>();if (!st_->open(f_, err)) return false;} else {- madvise(f_.map, f_.map_bytes, MADV_WILLNEED);+ { // read ahead the weights, not the n-gram table: in a Gyro file it is 28.8 GB in the middle of the one file, read by the host from+ // the SSD a few rows at a time (src/ngram/ple_reader.cpp), and must not fill the page cache+ const gguf::Tensor* ple = f_.find("per_layer_token_embd.weight");+ if (ple == nullptr) {+ madvise(f_.map, f_.map_bytes, MADV_WILLNEED);+ } else {+ const size_t page = 16384;+ const size_t lo = (size_t) ple->offset / page * page, hi = ((size_t) (ple->offset + ple->bytes) + page - 1) / page * page;+ if (lo > 0) madvise(f_.map, lo, MADV_WILLNEED);+ if (hi < f_.map_bytes) madvise((uint8_t*) f_.map + hi, f_.map_bytes - hi, MADV_WILLNEED);+ }+ }// the chunks of a model bigger than one Metal buffer overlap by the largest range bound at one offset: a tensor,// or a layer's routed experts (gate, up and down, bound together at the first of them: moe_rebase)size_t span = 0;@@ -593,6 +604,10 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionStasrc = raw.data();}ref::dequant(x.type, src, tmp.data(), ne);+ // a Q6_K / IQ4_NL matrix (Gyro) goes through BF16 first: the form the pack gives the CUDA engine+ // (tools/iq_pack.py --compat-bf16), so the Q8_0 copy below is made from the same numbers+ if (x.type != ref::BF16)+ for (uint64_t i = 0; i < ne; ++i) tmp[i] = ref::bf16_to_f32(ref::f32_to_bf16(tmp[i]));ref::quantize_q8_0(tmp.data(), ne, Q8_.as<uint8_t>() + q8off_[names[i]]);}});