Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Prefill PLE: opt-in STRATA_PLE_COMPACT=1 uploads the raw IQ4_NL rows and dequantizes them on the device (#3779, #3712)
linxuhao17b08c96339f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 31d3664b6..69a3fa7c0 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1428,6 +1428,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_test(NAME ple_parity COMMAND ple_parity --selftest--in "${STRATA_PLE_FIXTURE_DIR}/ple_in.bin" --out "${STRATA_PLE_FIXTURE_DIR}/ple_out.bin"WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})+ # STRATA_PLE_COMPACT: the raw IQ4_NL rows dequantized on the device against the host decode, bit for bit (synthetic, no model)+ add_executable(ple_iq4nl_dequant_test src/kernels/ple_iq4nl_dequant_test.cpp)+ target_include_directories(ple_iq4nl_dequant_test PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(ple_iq4nl_dequant_test PRIVATE strata_kernels ${_strata_gpu_runtime_target})+ add_test(NAME ple_iq4nl_dequant_test COMMAND ple_iq4nl_dequant_test)+ set_tests_properties(ple_iq4nl_dequant_test PROPERTIES SKIP_RETURN_CODE 77)# Plan v0.3 P7: INT8 KV storage vs a host reference and vs the FP16 path (GPU, synthetic, no model).add_executable(kv_q8_parity src/kernels/kv_q8_parity.cpp)diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hppindex d25a1eeec..9954a0f33 100644--- a/include/strata/kernels/ngram.hpp+++ b/include/strata/kernels/ngram.hpp@@ -187,6 +187,10 @@ public:/// as ONE reader request - page dedupe and sort across the whole batch, the reader's full queue depth. Not/// while a single-token `issue` is pending. The mapped mode gathers row by row.bool gather_batch(const uint32_t* rows, size_t n_tokens, float* out, std::string& err);+ /// STRATA_PLE_COMPACT: the same rows as their raw IQ4_NL bytes (n_tokens x 16 rows of PLE_ROW_BYTES; a row past the+ /// table: scale 0 and code 8 everywhere, which decode to the +0.0f gather_batch writes) for ple_iq4nl_dequant on the device. False (nothing+ /// written) unless the table is in memory (--ple-io ram / mmap) and IQ4_NL.+ bool gather_raw_iq4nl(const uint32_t* rows, size_t n_tokens, uint8_t* out) const;/// Fault injection (Direct mode): every row read completes no earlier than `us` after issue.void set_injected_delay_us(double us);diff --git a/include/strata/kernels/ple.hpp b/include/strata/kernels/ple.hppindex 98bfc388d..a558e1e27 100644--- a/include/strata/kernels/ple.hpp+++ b/include/strata/kernels/ple.hpp@@ -125,4 +125,8 @@ void ple_history_advance(float* hist, const float* normalized, void* stream);/// Whether the CUDA path is available. The host hash and the table read work without it; the block does not.bool ple_block_available();+/// STRATA_PLE_COMPACT: `rows` raw IQ4_NL PLE rows (PleTable::gather_raw_iq4nl, PLE_ROW_BYTES each) to 160 floats each,+/// the floats iq4nl_dequant_row writes (d times the code's value, one multiply), on `stream`.+void ple_iq4nl_dequant(const uint8_t* raw, int64_t rows, float* out, void* stream);+} // namespace strata::kernelsdiff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cppindex 0c1976f60..791d1df4a 100644--- a/metal/src/engine/stubs_0145.cpp+++ b/metal/src/engine/stubs_0145.cpp@@ -149,3 +149,5 @@ namespace strata::kernels { bool native_q8_0_sm60_pack(void const* p0, int p1, inamespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }// only reached with pf_wmma (off on Metal)namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }+// STRATA_PLE_COMPACT (opt-in, CUDA / HIP prompt path): prefill.cpp references it; the Metal prompt runs through its window and never reaches it.+namespace strata::kernels { void ple_iq4nl_dequant(const uint8_t* p0, int64_t p1, float* p2, void* p3) { strata::metal_engine::missing("strata::kernels::ple_iq4nl_dequant"); } }diff --git a/src/kernels/cuda/ple.cu b/src/kernels/cuda/ple.cuindex f945ef3b8..89c54502e 100644--- a/src/kernels/cuda/ple.cu+++ b/src/kernels/cuda/ple.cu@@ -23,6 +23,7 @@#include "strata/kernels/native_ple_postops.hpp"#include <cuda_runtime.h>+#include <cuda_fp16.h>#include <cmath>#include <cstdio>@@ -387,4 +388,26 @@ void ple_block_projected(const float* projected_key, const float* projected_valunative_ple_postops(projected_key, hidden, projected_value, hist_rows, w, buffers, stream);}+namespace {+__constant__ int8_t kPleIq4Nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};+// one thread a value: row r, block b (18 bytes: fp16 d, 16 bytes of codes), element j of 32 (j < 16: low nibble of+// qs[j], else the high nibble of qs[j - 16] - the split halves of iq4nl_dequant_row)+__global__ void ple_iq4nl_dequant_kernel(const uint8_t* __restrict__ raw, int64_t n, float* __restrict__ out) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ const int64_t r = i / PLE_HEAD_DIM;+ const int e = (int) (i - r * PLE_HEAD_DIM), b = e >> 5, j = e & 31;+ const uint8_t* blk = raw + r * PLE_ROW_BYTES + b * 18;+ const float d = __half2float(__ushort_as_half((unsigned short) (blk[0] | (blk[1] << 8))));+ const uint8_t q = blk[2 + (j & 15)];+ out[i] = d * (float) kPleIq4Nl[j < 16 ? (q & 15) : (q >> 4)];+}+} // namespace++void ple_iq4nl_dequant(const uint8_t* raw, int64_t rows, float* out, void* stream) {+ const int64_t n = rows * PLE_HEAD_DIM;+ if (n <= 0) return;+ ple_iq4nl_dequant_kernel<<<(unsigned) ((n + 255) / 256), 256, 0, (cudaStream_t) stream>>>(raw, n, out);+}+} // namespace strata::kernelsdiff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cppindex c32dc786a..7b8d844e7 100644--- a/src/kernels/ngram.cpp+++ b/src/kernels/ngram.cpp@@ -618,6 +618,26 @@ bool PleTable::gather_batch(const uint32_t* rows, size_t n_tokens, float* out, sreturn true;}+bool PleTable::gather_raw_iq4nl(const uint32_t* rows, size_t n_tokens, uint8_t* out) const {+ std::lock_guard<std::recursive_mutex> lk(impl_->mu);+ if (impl_->data == nullptr || impl_->fmt->id != PleFormat::IQ4_NL || impl_->rb != PLE_ROW_BYTES) return false;+ const size_t n = n_tokens * (size_t) PLE_N_HEADS;+ for (size_t i = 0; i < n; ++i) {+ uint8_t* o = out + i * PLE_ROW_BYTES;+ if (rows[i] >= impl_->n_rows) {+ // a row past the table reads as +0.0f (read_row): scale 0 and code 8 (value +1) in every nibble decode to +0.0f+ // on the device, where all-zero bytes would decode to -0.0f (scale 0 times code 0's value -127)+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b) {+ uint8_t* blk = o + (size_t) b * 18;+ std::memset(blk, 0, 2);+ std::memset(blk + 2, 0x88, 16);+ }+ } else std::memcpy(o, impl_->data + (size_t) rows[i] * PLE_ROW_BYTES, PLE_ROW_BYTES);+ }+ impl_->bytes_read += (uint64_t) n * PLE_ROW_BYTES;+ return true;+}+void PleTable::set_injected_delay_us(double us) { impl_->reader.set_injected_delay_us(us); }std::string PleTable::io_report() const {diff --git a/src/kernels/ple_iq4nl_dequant_test.cpp b/src/kernels/ple_iq4nl_dequant_test.cppnew file mode 100644index 000000000..e5312cf92--- /dev/null+++ b/src/kernels/ple_iq4nl_dequant_test.cpp@@ -0,0 +1,101 @@+// src/kernels/ple_iq4nl_dequant_test.cpp - STRATA_PLE_COMPACT: ple_iq4nl_dequant (raw IQ4_NL PLE rows to floats on the+// device) against iq4nl_dequant_row (the host decode the prompt path used before), bit for bit. Synthetic bytes, no model.+//+// - every one of the 65,536 fp16 scale patterns (infinities and NaNs included: those are compared as "both NaN") in+// every block position, with random codes;+// - rows of random bytes;+// - the row the table gives for an index past its end (scale 0, code 8 in every nibble: +0.0f, what PleTable::read_row+// writes for such a row);+// - an odd row count (not a multiple of the 256-thread block).+// Exit 77 without a CUDA device.+#include "strata/kernels/ngram.hpp"+#include "strata/kernels/ple.hpp"++#include <cuda_runtime.h>++#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstring>+#include <random>+#include <vector>++namespace {+using namespace strata::kernels;++int check(const std::vector<uint8_t>& raw, int64_t rows, const char* what) {+ float* dout = nullptr;+ uint8_t* draw = nullptr;+ if (cudaMalloc((void**) &dout, (size_t) rows * PLE_HEAD_DIM * 4) != cudaSuccess ||+ cudaMalloc((void**) &draw, raw.size()) != cudaSuccess ||+ cudaMemcpy(draw, raw.data(), raw.size(), cudaMemcpyHostToDevice) != cudaSuccess) {+ std::printf("CUDA setup failed for %s\n", what);+ return 1;+ }+ cudaMemset(dout, 0xff, (size_t) rows * PLE_HEAD_DIM * 4);+ ple_iq4nl_dequant(draw, rows, dout, nullptr);+ if (cudaDeviceSynchronize() != cudaSuccess) { std::printf("CUDA error for %s: %s\n", what, cudaGetErrorString(cudaGetLastError())); return 1; }+ std::vector<float> got((size_t) rows * PLE_HEAD_DIM), want(got.size());+ cudaMemcpy(got.data(), dout, got.size() * 4, cudaMemcpyDeviceToHost);+ cudaFree(dout);+ cudaFree(draw);+ for (int64_t r = 0; r < rows; ++r) iq4nl_dequant_row(raw.data() + (size_t) r * PLE_ROW_BYTES, want.data() + (size_t) r * PLE_HEAD_DIM);+ long long diff = 0, first = -1;+ for (size_t i = 0; i < got.size(); ++i) {+ uint32_t a, b;+ std::memcpy(&a, &got[i], 4);+ std::memcpy(&b, &want[i], 4);+ if (std::isnan(got[i]) && std::isnan(want[i])) continue; // a NaN scale: any payload is a NaN+ if (a != b) { if (first < 0) first = (long long) i; ++diff; }+ }+ std::printf(" %-52s %lld rows: %s\n", what, (long long) rows, diff ? "*** DIFFERS ***" : "bitwise equal");+ if (diff) std::printf(" %lld values differ, first at %lld\n", diff, first);+ return diff ? 1 : 0;+}+} // namespace++int main() {+ int n_dev = 0;+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) { std::printf("no CUDA device (skipped)\n"); return 77; }+ int bad = 0;+ std::mt19937 rng(20261026u);+ { // all fp16 patterns: 65,536 scales over 5 blocks a row = 13,108 rows (the last row pads with 0)+ const int64_t rows = (65536 + 4) / 5;+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);+ for (auto& b : raw) b = (uint8_t) (rng() & 0xff);+ uint32_t d = 0;+ for (int64_t r = 0; r < rows; ++r)+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b, ++d) {+ const uint16_t h = (uint16_t) (d & 0xffff);+ std::memcpy(&raw[(size_t) r * PLE_ROW_BYTES + (size_t) b * 18], &h, 2);+ }+ bad += check(raw, rows, "every fp16 scale pattern, random codes");+ }+ { // random rows+ const int64_t rows = 4096 + 37;+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);+ for (auto& b : raw) b = (uint8_t) (rng() & 0xff);+ bad += check(raw, rows, "random bytes (odd row count)");+ }+ { // the out-of-table row of PleTable::gather_raw_iq4nl+ const int64_t rows = 16;+ std::vector<uint8_t> raw((size_t) rows * PLE_ROW_BYTES);+ for (int64_t r = 0; r < rows; ++r)+ for (int b = 0; b < PLE_HEAD_DIM / 32; ++b) {+ uint8_t* blk = &raw[(size_t) r * PLE_ROW_BYTES + (size_t) b * 18];+ std::memset(blk, 0, 2);+ std::memset(blk + 2, 0x88, 16);+ }+ bad += check(raw, rows, "out-of-table row (scale 0, code 8)");+ // and it is +0.0f, not -0.0f, on both sides+ std::vector<float> want(PLE_HEAD_DIM);+ iq4nl_dequant_row(raw.data(), want.data());+ for (float v : want) {+ uint32_t a;+ std::memcpy(&a, &v, 4);+ if (a != 0) { std::printf(" *** the out-of-table row does not decode to +0.0f\n"); ++bad; break; }+ }+ }+ std::printf("ple_iq4nl_dequant_test: %s\n", bad ? "FAILED" : "OK");+ return bad ? 1 : 0;+}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex a96c2a03d..a101503f0 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -132,6 +132,14 @@ bool prompt_run(const int64_t* tokens, int64_t n, int64_t pos0, const float* con} // namespace strata::metal_engine#endif+namespace {+// STRATA_PLE_COMPACT=1 (opt-in): the prompt chunk's PLE rows go up raw and are dequantized on the device (IQ4_NL tables+// held in memory); the same floats as the host dequant, 7x fewer bytes over PCIe+bool ple_compact_on() {+ static const bool on = [] { const char* v = std::getenv("STRATA_PLE_COMPACT"); return v != nullptr && v[0] == '1'; }();+ return on;+}+} // namespacenamespace strata::prefill {namespace {@@ -987,6 +995,8 @@ struct Prefill::Impl {float* ple_emb_host[2] = {}; // pinned, double-buffered: the next chunk's rows are read while thiscudaEvent_t ple_copied[2] = {}; // one runs; the event marks that buffer's upload donestd::vector<uint32_t> ple_rows[2];+ bool ple_compact[2] = {false, false}; // STRATA_PLE_COMPACT: the pinned buffer holds raw IQ4_NL rows+ uint8_t* ple_raw_dev = nullptr; // their device copy (cudaMalloc'd, the chunk's 16 rows a token)float* ple_norm = nullptr;uint8_t* region = nullptr; // the attention/MoE scratch region (idle while the PLE block runs)uint64_t region_bytes = 0;@@ -1097,6 +1107,7 @@ void Prefill::release() {if (e) cudaEventDestroy(e);if (impl_->ple_copied[b]) cudaEventDestroy(impl_->ple_copied[b]);if (impl_->ple_emb_host[b] && impl_->ple_pageable[b].empty()) cudaFreeHost(impl_->ple_emb_host[b]);+ if (b == 0 && impl_->ple_raw_dev) { (void) cudaFree(impl_->ple_raw_dev); impl_->ple_raw_dev = nullptr; }}if (impl_->copy) cudaStreamDestroy(impl_->copy);if (impl_->kv_copy) cudaStreamDestroy(impl_->kv_copy);@@ -1525,6 +1536,11 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, coif (!m.ple_copied[b] && cudaEventCreateWithFlags(&m.ple_copied[b], cudaEventDisableTiming) != cudaSuccess)ok = false;m.ple_rows[b].resize(T * strata::kernels::PLE_N_HEADS);+ if (ple_compact_on() && m.ple_raw_dev == nullptr &&+ cudaMalloc((void**) &m.ple_raw_dev, (size_t) T * strata::kernels::PLE_N_HEADS * strata::kernels::PLE_ROW_BYTES) != cudaSuccess) {+ m.ple_raw_dev = nullptr;+ (void) cudaGetLastError();+ }}if (ss.qsa_states[ss.qsa_primary()].kv_mode == 1) { // KV streaming: the staging pool's identity page tableconst int64_t pages = ss.qsa_states[ss.qsa_primary()].n_pages;@@ -2839,6 +2855,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stripv[0] = pv[1];pv[1] = tok;}+ // STRATA_PLE_COMPACT=1: the raw rows (1,440 bytes a token instead of 10,240 floats' bytes) into the same pinned+ // buffer; ple_land uploads them and dequantizes on the device+ if (ple_compact_on() && m.ple_pageable[buf].empty() && m.ple_raw_dev != nullptr &&+ ss.ple.table->gather_raw_iq4nl(m.ple_rows[buf].data(), (size_t) T, (uint8_t*) m.ple_emb_host[buf])) {+ m.ple_compact[buf] = true;+ return true;+ }+ m.ple_compact[buf] = false;return ss.ple.table->gather_batch(m.ple_rows[buf].data(), (size_t) T, m.ple_emb_host[buf], e);};std::string ple_next_err;@@ -3002,7 +3026,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strireturn false;}// #1996: the pageable fallback (no RAM to pin) goes up through a pinned slot; the pinned buffers as before- if ((m.ple_pageable[ple_buf].empty()+ const size_t raw_bytes = (size_t) T * strata::kernels::PLE_N_HEADS * strata::kernels::PLE_ROW_BYTES;+ if ((m.ple_compact[ple_buf]+ ? cudaMemcpyAsync(m.ple_raw_dev, m.ple_emb_host[ple_buf], raw_bytes, cudaMemcpyHostToDevice, m.cs)+ : m.ple_pageable[ple_buf].empty()? cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, m.cs): m.up_ring.h2d(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, m.cs)) !=cudaSuccess ||@@ -3010,6 +3037,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strierr = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());return false;}+ if (m.ple_compact[ple_buf])+ strata::kernels::ple_iq4nl_dequant(m.ple_raw_dev, T * strata::kernels::PLE_N_HEADS, m.ple_emb, m.cs);if (c0 + C < n) {// the other buffer's upload (a chunk ago) is done before the SSD thread refills itif (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) {